diff options
| author | android-build-team Robot <android-build-team-robot@google.com> | 2021-06-21 14:49:04 +0000 |
|---|---|---|
| committer | android-build-team Robot <android-build-team-robot@google.com> | 2021-06-21 14:49:04 +0000 |
| commit | b6ad432c2d11260738e59080088d73b1aab40da9 (patch) | |
| tree | 6a87a8e100def2a9eb90995776a41c42d6045331 | |
| parent | c3d97aa4219e73aaaad86ce6a8c2f69600871bc6 (diff) | |
| parent | 69049fa8fcf8528333913cd220587c71b195ea27 (diff) | |
| download | pcre-b6ad432c2d11260738e59080088d73b1aab40da9.tar.gz | |
Snap for 7478067 from 69049fa8fcf8528333913cd220587c71b195ea27 to mainline-extservices-releaseandroid-mainline-12.0.0_r9android-mainline-12.0.0_r87android-mainline-12.0.0_r72android-mainline-12.0.0_r44android-mainline-12.0.0_r27android-mainline-12.0.0_r117android-mainline-12.0.0_r102android12-mainline-extservices-release
Change-Id: Ifc8f1c951ca842b0ff23c2fa518207afcb297fca
189 files changed, 27770 insertions, 27188 deletions
@@ -1,5 +1,38 @@ +package { + default_applicable_licenses: ["external_pcre_license"], +} + +// Added automatically by a large-scale-change that took the approach of +// 'apply every license found to every target'. While this makes sure we respect +// every license restriction, it may not be entirely correct. +// +// e.g. GPL in an MIT project might only apply to the contrib/ directory. +// +// Please consider splitting the single license below into multiple licenses, +// taking care not to lose any license_kind information, and overriding the +// default license using the 'licenses: [...]' property on targets as needed. +// +// For unused files, consider creating a 'fileGroup' with "//visibility:private" +// to attach the license to, and including a comment whether the files may be +// used in the current project. +// See: http://go/android-license-faq +license { + name: "external_pcre_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-BSD", + "SPDX-license-identifier-FSFAP", + "SPDX-license-identifier-MIT", + "legacy_notice", + "legacy_unencumbered", + ], + license_text: [ + "NOTICE", + ], +} + libpcre2_dist_prefix = "dist2" libpcre2_src_files = [ @@ -59,10 +92,13 @@ cc_library { defaults: ["pcre_defaults"], host_supported: true, vendor_available: true, + product_available: true, vndk: { enabled: true, }, double_loadable: true, + ramdisk_available: true, + vendor_ramdisk_available: true, recovery_available: true, srcs: libpcre2_src_files, stl: "none", @@ -10,6 +10,7 @@ third_party { value: "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-10.33.zip" } version: "10.33" + license_type: NOTICE last_upgrade_date { year: 2019 month: 8 diff --git a/dist2/AUTHORS b/dist2/AUTHORS index 8d4e15a2..f001cb77 100644 --- a/dist2/AUTHORS +++ b/dist2/AUTHORS @@ -2,13 +2,13 @@ THE MAIN PCRE2 LIBRARY CODE --------------------------- Written by: Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk +Email local part: Philip.Hazel +Email domain: gmail.com University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2019 University of Cambridge +Copyright (c) 1997-2020 University of Cambridge All rights reserved @@ -19,7 +19,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2010-2019 Zoltan Herczeg +Copyright(c) 2010-2020 Zoltan Herczeg All rights reserved. @@ -30,7 +30,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2009-2019 Zoltan Herczeg +Copyright(c) 2009-2020 Zoltan Herczeg All rights reserved. #### diff --git a/dist2/CMakeLists.txt b/dist2/CMakeLists.txt index a7ef67e1..aec29796 100644 --- a/dist2/CMakeLists.txt +++ b/dist2/CMakeLists.txt @@ -1,6 +1,5 @@ # CMakeLists.txt # -# # This file enables PCRE2 to be built with the CMake configuration and build # tool. Download CMake in source or binary form from http://www.cmake.org/ # Converted to support PCRE2 from the original PCRE file, August 2014. @@ -85,19 +84,33 @@ # 2018-11-14 PH removed unnecessary checks for stdint.h and inttypes.h # 2018-11-16 PH added PCRE2GREP_SUPPORT_CALLOUT_FORK support and tidied # 2019-02-16 PH hacked to avoid CMP0026 policy issue (see comments below) +# 2020-03-16 PH renamed dftables as pcre2_dftables (as elsewhere) +# 2020-03-24 PH changed CMAKE_MODULE_PATH definition to add, not replace +# 2020-04-08 Carlo added function check for secure_getenv, fixed strerror +# 2020-04-16 enh added check for __attribute__((uninitialized)) +# 2020-04-25 PH applied patches from Uwe Korn to support pkg-config and +# library versioning. +# 2020-04-25 Carlo added function check for mkostemp used in ProtExecAllocator +# 2020-04-28 PH added function check for memfd_create based on Carlo's patch +# 2020-05-25 PH added a check for Intel CET +# 2020-12-03 PH altered the definition of pcre2test as suggested by Daniel PROJECT(PCRE2 C) -# Increased minimum to 2.8.0 to support newer add_test features. -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.0) +# Increased minimum to 2.8.5 to support GNUInstallDirs. +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.5) # Set policy CMP0026 to avoid warnings for the use of LOCATION in # GET_TARGET_PROPERTY. This should no longer be required. # CMAKE_POLICY(SET CMP0026 OLD) -SET(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) # for FindReadline.cmake +# For FindReadline.cmake. This was changed to allow setting CMAKE_MODULE_PATH +# on the command line. +# SET(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) + +LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I${PROJECT_SOURCE_DIR}/src") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I\"${PROJECT_SOURCE_DIR}/src\"") # external packages FIND_PACKAGE( BZip2 ) @@ -109,8 +122,10 @@ FIND_PACKAGE( Editline ) INCLUDE(CheckCSourceCompiles) INCLUDE(CheckFunctionExists) +INCLUDE(CheckSymbolExists) INCLUDE(CheckIncludeFile) INCLUDE(CheckTypeSize) +INCLUDE(GNUInstallDirs) # for CMAKE_INSTALL_LIBDIR CHECK_INCLUDE_FILE(dirent.h HAVE_DIRENT_H) CHECK_INCLUDE_FILE(stdint.h HAVE_STDINT_H) @@ -120,9 +135,11 @@ CHECK_INCLUDE_FILE(sys/types.h HAVE_SYS_TYPES_H) CHECK_INCLUDE_FILE(unistd.h HAVE_UNISTD_H) CHECK_INCLUDE_FILE(windows.h HAVE_WINDOWS_H) -CHECK_FUNCTION_EXISTS(bcopy HAVE_BCOPY) -CHECK_FUNCTION_EXISTS(memmove HAVE_MEMMOVE) -CHECK_FUNCTION_EXISTS(strerror HAVE_STRERROR) +CHECK_SYMBOL_EXISTS(bcopy "strings.h" HAVE_BCOPY) +CHECK_SYMBOL_EXISTS(memfd_create "sys/mman.h" HAVE_MEMFD_CREATE) +CHECK_SYMBOL_EXISTS(memmove "string.h" HAVE_MEMMOVE) +CHECK_SYMBOL_EXISTS(secure_getenv "stdlib.h" HAVE_SECURE_GETENV) +CHECK_SYMBOL_EXISTS(strerror "string.h" HAVE_STRERROR) set(ORIG_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror") @@ -132,6 +149,24 @@ CHECK_C_SOURCE_COMPILES( ) set(CMAKE_REQUIRED_FLAGS ${ORIG_CMAKE_REQUIRED_FLAGS}) +# Check whether Intel CET is enabled, and if so, adjust compiler flags. This +# code was written by PH, trying to imitate the logic from the autotools +# configuration. + +CHECK_C_SOURCE_COMPILES( + "#ifndef __CET__ + #error CET is not enabled + #endif + int main() { return 0; }" + INTEL_CET_ENABLED +) + +IF (INTEL_CET_ENABLED) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mshstk") +ENDIF(INTEL_CET_ENABLED) + + + # User-configurable options # # Note: CMakeSetup displays these in alphabetical order, regardless of @@ -186,8 +221,12 @@ SET(PCRE2_HEAP_MATCH_RECURSE OFF CACHE BOOL SET(PCRE2_SUPPORT_JIT OFF CACHE BOOL "Enable support for Just-in-time compiling.") -SET(PCRE2_SUPPORT_JIT_SEALLOC OFF CACHE BOOL - "Enable SELinux compatible execmem allocator in JIT.") +IF(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) + SET(PCRE2_SUPPORT_JIT_SEALLOC OFF CACHE BOOL + "Enable SELinux compatible execmem allocator in JIT (experimental).") +ELSE(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) + SET(PCRE2_SUPPORT_JIT_SEALLOC IGNORE) +ENDIF(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) SET(PCRE2GREP_SUPPORT_JIT ON CACHE BOOL "Enable use of Just-in-time compiling in pcre2grep.") @@ -313,7 +352,19 @@ IF(PCRE2_SUPPORT_JIT) ENDIF(PCRE2_SUPPORT_JIT) IF(PCRE2_SUPPORT_JIT_SEALLOC) - SET(SLJIT_PROT_EXECUTABLE_ALLOCATOR 1) + SET(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) + CHECK_SYMBOL_EXISTS(mkostemp stdlib.h REQUIRED) + UNSET(CMAKE_REQUIRED_DEFINITIONS) + IF(${REQUIRED}) + IF(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) + ADD_DEFINITIONS(-D_GNU_SOURCE) + SET(SLJIT_PROT_EXECUTABLE_ALLOCATOR 1) + ELSE(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) + MESSAGE(FATAL_ERROR "Your configuration is not supported") + ENDIF(${CMAKE_SYSTEM_NAME} MATCHES Linux|NetBSD) + ELSE(${REQUIRED}) + SET(PCRE2_SUPPORT_JIT_SEALLOC OFF) + ENDIF(${REQUIRED}) ENDIF(PCRE2_SUPPORT_JIT_SEALLOC) IF(PCRE2GREP_SUPPORT_JIT) @@ -409,12 +460,13 @@ file(STRINGS ${PROJECT_SOURCE_DIR}/configure.ac LIMIT_COUNT 50 # Read only the first 50 lines of the file ) -set(SEARCHED_VARIABLES "pcre2_major" "pcre2_minor" "pcre2_prerelease" "pcre2_date") +set(SEARCHED_VARIABLES "pcre2_major" "pcre2_minor" "pcre2_prerelease" "pcre2_date" + "libpcre2_posix_version" "libpcre2_8_version" "libpcre2_16_version" "libpcre2_32_version") foreach(configure_line ${configure_lines}) foreach(_substitution_variable ${SEARCHED_VARIABLES}) string(TOUPPER ${_substitution_variable} _substitution_variable_upper) if (NOT ${_substitution_variable_upper}) - string(REGEX MATCH "m4_define\\(${_substitution_variable}, \\[(.*)\\]" MACTHED_STRING ${configure_line}) + string(REGEX MATCH "m4_define\\(${_substitution_variable}, *\\[(.*)\\]" MATCHED_STRING ${configure_line}) if (CMAKE_MATCH_1) set(${_substitution_variable_upper} ${CMAKE_MATCH_1}) endif() @@ -422,21 +474,83 @@ foreach(configure_line ${configure_lines}) endforeach() endforeach() +macro(PARSE_LIB_VERSION VARIABLE_PREFIX) + string(REPLACE ":" ";" ${VARIABLE_PREFIX}_VERSION_LIST ${${VARIABLE_PREFIX}_VERSION}) + list(GET ${VARIABLE_PREFIX}_VERSION_LIST 0 ${VARIABLE_PREFIX}_VERSION_CURRENT) + list(GET ${VARIABLE_PREFIX}_VERSION_LIST 1 ${VARIABLE_PREFIX}_VERSION_REVISION) + list(GET ${VARIABLE_PREFIX}_VERSION_LIST 2 ${VARIABLE_PREFIX}_VERSION_AGE) + + math(EXPR ${VARIABLE_PREFIX}_SOVERSION "${${VARIABLE_PREFIX}_VERSION_CURRENT} - ${${VARIABLE_PREFIX}_VERSION_AGE}") + math(EXPR ${VARIABLE_PREFIX}_MACHO_COMPATIBILITY_VERSION "${${VARIABLE_PREFIX}_VERSION_CURRENT} + 1") + math(EXPR ${VARIABLE_PREFIX}_MACHO_CURRENT_VERSION "${${VARIABLE_PREFIX}_VERSION_CURRENT} + 1") + set(${VARIABLE_PREFIX}_MACHO_CURRENT_VERSION "${${VARIABLE_PREFIX}_MACHO_CURRENT_VERSION}.${${VARIABLE_PREFIX}_VERSION_REVISION}}") + set(${VARIABLE_PREFIX}_VERSION "${${VARIABLE_PREFIX}_SOVERSION}.${${VARIABLE_PREFIX}_VERSION_AGE}.${${VARIABLE_PREFIX}_VERSION_REVISION}") +endmacro() + +PARSE_LIB_VERSION(LIBPCRE2_POSIX) +PARSE_LIB_VERSION(LIBPCRE2_8) +PARSE_LIB_VERSION(LIBPCRE2_16) +PARSE_LIB_VERSION(LIBPCRE2_32) + CONFIGURE_FILE(src/pcre2.h.in ${PROJECT_BINARY_DIR}/pcre2.h @ONLY) -# What about pcre2-config and libpcre2.pc? +# Make sure to not link debug libs +# against release libs and vice versa +IF(WIN32) + SET(CMAKE_DEBUG_POSTFIX "d") +ENDIF(WIN32) + +# Generate pkg-config files + +SET(PACKAGE_VERSION "${PCRE2_MAJOR}.${PCRE2_MINOR}") +SET(prefix ${CMAKE_INSTALL_PREFIX}) + +SET(exec_prefix "\${prefix}") +SET(libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}") +SET(includedir "\${prefix}/include") +IF(WIN32 AND (CMAKE_BUILD_TYPE MATCHES Debug)) + SET(LIB_POSTFIX ${CMAKE_DEBUG_POSTFIX}) +ENDIF() +CONFIGURE_FILE(libpcre2-posix.pc.in libpcre2-posix.pc @ONLY) +SET(pkg_config_files ${pkg_config_files} "${CMAKE_CURRENT_BINARY_DIR}/libpcre2-posix.pc") + +IF(PCRE2_BUILD_PCRE2_8) + CONFIGURE_FILE(libpcre2-8.pc.in libpcre2-8.pc @ONLY) + SET(pkg_config_files ${pkg_config_files} "${CMAKE_CURRENT_BINARY_DIR}/libpcre2-8.pc") + SET(enable_pcre2_8 "yes") +ELSE() + SET(enable_pcre2_8 "no") +ENDIF() + +IF(PCRE2_BUILD_PCRE2_16) + CONFIGURE_FILE(libpcre2-16.pc.in libpcre2-16.pc @ONLY) + SET(pkg_config_files ${pkg_config_files} "${CMAKE_CURRENT_BINARY_DIR}/libpcre2-16.pc") + SET(enable_pcre2_16 "yes") +ELSE() + SET(enable_pcre2_16 "no") +ENDIF() + +IF(PCRE2_BUILD_PCRE2_32) + CONFIGURE_FILE(libpcre2-32.pc.in libpcre2-32.pc @ONLY) + SET(pkg_config_files ${pkg_config_files} "${CMAKE_CURRENT_BINARY_DIR}/libpcre2-32.pc") + SET(enable_pcre2_32 "yes") +ELSE() + SET(enable_pcre2_32 "no") +ENDIF() + +CONFIGURE_FILE(pcre2-config.in pcre2-config @ONLY) # Character table generation OPTION(PCRE2_REBUILD_CHARTABLES "Rebuild char tables" OFF) IF(PCRE2_REBUILD_CHARTABLES) - ADD_EXECUTABLE(dftables src/dftables.c) + ADD_EXECUTABLE(pcre2_dftables src/pcre2_dftables.c) ADD_CUSTOM_COMMAND( COMMENT "Generating character tables (pcre2_chartables.c) for current locale" - DEPENDS dftables - COMMAND dftables + DEPENDS pcre2_dftables + COMMAND pcre2_dftables ARGS ${PROJECT_BINARY_DIR}/pcre2_chartables.c OUTPUT ${PROJECT_BINARY_DIR}/pcre2_chartables.c ) @@ -538,11 +652,6 @@ IF(MSVC) ENDIF(MSVC) SET(CMAKE_INCLUDE_CURRENT_DIR 1) -# needed to make sure to not link debug libs -# against release libs and vice versa -IF(WIN32) - SET(CMAKE_DEBUG_POSTFIX "d") -ENDIF(WIN32) SET(targets) @@ -550,12 +659,20 @@ SET(targets) IF(PCRE2_BUILD_PCRE2_8) ADD_LIBRARY(pcre2-8 ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -SET_PROPERTY(TARGET pcre2-8 - PROPERTY COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8) +SET_TARGET_PROPERTIES(pcre2-8 PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_8_MACHO_COMPATIBILITY_VERSION}" + MACHO_CURRENT_VERSION "${LIBPCRE2_8_MACHO_CURRENT_VERSION}" + VERSION ${LIBPCRE2_8_VERSION} + SOVERSION ${LIBPCRE2_8_SOVERSION}) SET(targets ${targets} pcre2-8) ADD_LIBRARY(pcre2-posix ${PCRE2POSIX_HEADERS} ${PCRE2POSIX_SOURCES}) -SET_PROPERTY(TARGET pcre2-posix - PROPERTY COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8) +SET_TARGET_PROPERTIES(pcre2-posix PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_POSIX_MACHO_COMPATIBILITY_VERSION}" + MACHO_CURRENT_VERSION "${LIBPCRE2_POSIX_MACHO_CURRENT_VERSION}" + VERSION ${LIBPCRE2_POSIX_VERSION} + SOVERSION ${LIBPCRE2_POSIX_SOVERSION}) SET(targets ${targets} pcre2-posix) TARGET_LINK_LIBRARIES(pcre2-posix pcre2-8) @@ -573,8 +690,12 @@ ENDIF(PCRE2_BUILD_PCRE2_8) IF(PCRE2_BUILD_PCRE2_16) ADD_LIBRARY(pcre2-16 ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -SET_PROPERTY(TARGET pcre2-16 - PROPERTY COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16) +SET_TARGET_PROPERTIES(pcre2-16 PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" + MACHO_CURRENT_VERSION "${LIBPCRE2_32_MACHO_CURRENT_VERSION}" + VERSION ${LIBPCRE2_16_VERSION} + SOVERSION ${LIBPCRE2_16_SOVERSION}) SET(targets ${targets} pcre2-16) IF(MINGW AND NOT PCRE2_STATIC) @@ -591,8 +712,12 @@ ENDIF(PCRE2_BUILD_PCRE2_16) IF(PCRE2_BUILD_PCRE2_32) ADD_LIBRARY(pcre2-32 ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -SET_PROPERTY(TARGET pcre2-32 - PROPERTY COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32) +SET_TARGET_PROPERTIES(pcre2-32 PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" + MACHO_CURRENT_VERSION "${LIBPCRE2_32_MACHO_CURRENT_VERSION}" + VERSION ${LIBPCRE2_32_VERSION} + SOVERSION ${LIBPCRE2_32_SOVERSION}) SET(targets ${targets} pcre2-32) IF(MINGW AND NOT PCRE2_STATIC) @@ -727,7 +852,9 @@ if test \"$?\" != \"0\"; then exit 1; fi \@echo off setlocal SET srcdir=\"${winsrc}\" -SET pcre2test=\"${winexe}\" +# The next line was replaced by the following one after a user comment. +# SET pcre2test=\"${winexe}\" +SET pcre2test=\"${winbin}\\pcre2test.exe\" if not [%CMAKE_CONFIG_TYPE%]==[] SET pcre2test=\"${winbin}\\%CMAKE_CONFIG_TYPE%\\pcre2test.exe\" call %srcdir%\\RunTest.Bat if errorlevel 1 exit /b 1 @@ -763,8 +890,13 @@ SET(CMAKE_INSTALL_ALWAYS 1) INSTALL(TARGETS ${targets} RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +INSTALL(FILES ${pkg_config_files} DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/pcre2-config" + DESTINATION bin + # Set 0755 permissions + PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) INSTALL(FILES ${PCRE2_HEADERS} ${PCRE2POSIX_HEADERS} DESTINATION include) @@ -811,7 +943,7 @@ IF(PCRE2_SHOW_REPORT) ENDIF(CMAKE_C_FLAGS) MESSAGE(STATUS "") MESSAGE(STATUS "") - MESSAGE(STATUS "PCRE2 configuration summary:") + MESSAGE(STATUS "PCRE2-${PCRE2_MAJOR}.${PCRE2_MINOR} configuration summary:") MESSAGE(STATUS "") MESSAGE(STATUS " Install prefix .................. : ${CMAKE_INSTALL_PREFIX}") MESSAGE(STATUS " C compiler ...................... : ${CMAKE_C_COMPILER}") diff --git a/dist2/ChangeLog b/dist2/ChangeLog index 66c6d0b3..2e20bdbb 100644 --- a/dist2/ChangeLog +++ b/dist2/ChangeLog @@ -1,6 +1,417 @@ Change Log for PCRE2 -------------------- +Version 10.36-RC1 04-December-2020 +---------------------------------- + +1. Add CET_CFLAGS so that when Intel CET is enabled, pass -mshstk to +compiler. This fixes https://bugs.exim.org/show_bug.cgi?id=2578. Patch for +Makefile.am and configure.ac by H.J. Lu. Equivalent patch for CMakeLists.txt +invented by PH. + +2. Fix inifinite loop when a single byte newline is searched in JIT when +invalid utf8 mode is enabled. + +3. Updated CMakeLists.txt with patch from Wolfgang Stöggl (Bugzilla #2584): + + - Include GNUInstallDirs and use ${CMAKE_INSTALL_LIBDIR} instead of hardcoded + lib. This allows differentiation between lib and lib64. + CMAKE_INSTALL_LIBDIR is used for installation of libraries and also for + pkgconfig file generation. + + - Add the version of PCRE2 to the configuration summary like ./configure + does. + + - Fix typo: MACTHED_STRING->MATCHED_STRING + +4. Updated CMakeLists.txt with another patch from Wolfgang Stöggl (Bugzilla +#2588): + + - Add escaped double quotes around include directory in CMakeLists.txt to + allow spaces in directory names. + + - This fixes a cmake error, if the path of the pcre2 source contains a space. + +5. Updated CMakeLists.txt with a patch from B. Scott Michel: CMake's +documentation suggests using CHECK_SYMBOL_EXISTS over CHECK_FUNCTION_EXIST. +Moreover, these functions come from specific header files, which need to be +specified (and, thankfully, are the same on both the Linux and WinXX +platforms.) + +6. Added a (uint32_t) cast to prevent a compiler warning in pcre2_compile.c. + +7. Applied a patch from Wolfgang Stöggl (Bugzilla #2600) to fix postfix for +debug Windows builds using CMake. This also updated configure so that it +generates *.pc files and pcre2-config with the same content, as in the past. + +8. If a pattern ended with (?(VERSION=n.d where n is any number but d is just a +single digit, the code unit beyond d was being read (i.e. there was a read +buffer overflow). Fixes ClusterFuzz 23779. + +9. After the rework in r1235, certain character ranges were incorrectly +handled by an optimization in JIT. Furthermore a wrong offset was used to +read a value from a buffer which could lead to memory overread. + +10. Unnoticed for many years was the fact that delimiters other than / in the +testinput1 and testinput4 files could cause incorrect behaviour when these +files were processed by perltest.sh. There were several tests that used quotes +as delimiters, and it was just luck that they didn't go wrong with perltest.sh. +All the patterns in testinput1 and testinput4 now use / as their delimiter. +This fixes Bugzilla #2641. + +11. Perl has started to give an error for \K within lookarounds (though there +are cases where it doesn't). PCRE2 still allows this, so the tests that include +this case have been moved from test 1 to test 2. + +12. Further to 10 above, pcre2test has been updated to detect and grumble if a +delimiter other than / is used after #perltest. + +13. Fixed a bug with PCRE2_MATCH_INVALID_UTF in 8-bit mode when PCRE2_CASELESS +was set and PCRE2_NO_START_OPTIMIZE was not set. The optimization for finding +the start of a match was not resetting correctly after a failed match on the +first valid fragment of the subject, possibly causing incorrect "no match" +returns on subsequent fragments. For example, the pattern /A/ failed to match +the subject \xe5A. Fixes Bugzilla #2642. + +14. Fixed a bug in character set matching when JIT is enabled and both unicode +scripts and unicode classes are present at the same time. + +15. Added GNU grep's -m (aka --max-count) option to pcre2grep. + +16. Refactored substitution processing in pcre2grep strings, both for the -O +option and when dealing with callouts. There is now a single function that +handles $ expansion in all cases (instead of multiple copies of almost +identical code). This means that the same escape sequences are available +everywhere, which was not previously the case. At the same time, the escape +sequences $x{...} and $o{...} have been introduced, to allow for characters +whose code points are greater than 255 in Unicode mode. + +17. Applied the patch from Bugzilla #2628 to RunGrepTest. This does an explicit +test for a version of sed that can handle binary zero, instead of assuming that +any Linux version will work. Later: replaced $(...) by `...` because not all +shells recognize the former. + +18. Fixed a word boundary check bug in JIT when partial matching is enabled. + +19. Fix ARM64 compilation warning in JIT. Patch by Carlo. + +20. A bug in the RunTest script meant that if the first part of test 2 failed, +the failure was not reported. + +21. Test 2 was failing when run from a directory other than the source +directory. This failure was previously missed in RunTest because of 20 above. +Fixes added to both RunTest and RunTest.bat. + +22. Patch to CMakeLists.txt from Daniel to fix problem with testing under +Windows. + + +Version 10.35 09-May-2020 +--------------------------- + +1. Use PCRE2_MATCH_EMPTY flag to detect empty matches in JIT. + +2. Fix ARMv5 JIT improper handling of labels right after a constant pool. + +3. A JIT bug is fixed which allowed to read the fields of the compiled +pattern before its existence is checked. + +4. Back in the PCRE1 day, capturing groups that contained recursive back +references to themselves were made atomic (version 8.01, change 18) because +after the end a repeated group, the captured substrings had their values from +the final repetition, not from an earlier repetition that might be the +destination of a backtrack. This feature was documented, and was carried over +into PCRE2. However, it has now been realized that the major refactoring that +was done for 10.30 has made this atomicizing unnecessary, and it is confusing +when users are unaware of it, making some patterns appear not to be working as +expected. Capture values of recursive back references in repeated groups are +now correctly backtracked, so this unnecessary restriction has been removed. + +5. Added PCRE2_SUBSTITUTE_LITERAL. + +6. Avoid some VS compiler warnings. + +7. Added PCRE2_SUBSTITUTE_MATCHED. + +8. Added (?* and (?<* as synonms for (*napla: and (*naplb: to match another +regex engine. The Perl regex folks are aware of this usage and have made a note +about it. + +9. When an assertion is repeated, PCRE2 used to limit the maximum repetition to +1, believing that repeating an assertion is pointless. However, if a positive +assertion contains capturing groups, repetition can be useful. In any case, an +assertion could always be wrapped in a repeated group. The only restriction +that is now imposed is that an unlimited maximum is changed to one more than +the minimum. + +10. Fix *THEN verbs in lookahead assertions in JIT. + +11. Added PCRE2_SUBSTITUTE_REPLACEMENT_ONLY. + +12. The JIT stack should be freed when the low-level stack allocation fails. + +13. In pcre2grep, if the final line in a scanned file is output but does not +end with a newline sequence, add a newline according to the --newline setting. + +14. (?(DEFINE)...) groups were not being handled correctly when checking for +the fixed length of a lookbehind assertion. Such a group within a lookbehind +should be skipped, as it does not contribute to the length of the group. +Instead, the (DEFINE) group was being processed, and if at the end of the +lookbehind, that end was not correctly recognized. Errors such as "lookbehind +assertion is not fixed length" and also "internal error: bad code value in +parsed_skip()" could result. + +15. Put a limit of 1000 on recursive calls in pcre2_study() when searching +nested groups for starting code units, in order to avoid stack overflow issues. +If the limit is reached, it just gives up trying for this optimization. + +16. The control verb chain list must always be restored when exiting from a +recurse function in JIT. + +17. Fix a crash which occurs when the character type of an invalid UTF +character is decoded in JIT. + +18. Changes in many areas of the code so that when Unicode is supported and +PCRE2_UCP is set without PCRE2_UTF, Unicode character properties are used for +upper/lower case computations on characters whose code points are greater than +127. + +19. The function for checking UTF-16 validity was returning an incorrect offset +for the start of the error when a high surrogate was not followed by a valid +low surrogate. This caused incorrect behaviour, for example when +PCRE2_MATCH_INVALID_UTF was set and a match started immediately following the +invalid high surrogate, such as /aa/ matching "\x{d800}aa". + +20. If a DEFINE group immediately preceded a lookbehind assertion, the pattern +could be mis-compiled and therefore not match correctly. This is the example +that found this: /(?(DEFINE)(?<foo>bar))(?<![-a-z0-9])word/ which failed to +match "word" because the "move back" value was set to zero. + +21. Following a request from a user, some extensions and tidies to the +character tables handling have been done: + + (a) The dftables auxiliary program is renamed pcre2_dftables, but it is still + not installed for public use. + + (b) There is now a -b option for pcre2_dftables, which causes the tables to + be written in binary. There is also a -help option. + + (c) PCRE2_CONFIG_TABLES_LENGTH is added to pcre2_config() so that an + application that wants to save tables in binary knows how long they are. + +22. Changed setting of CMAKE_MODULE_PATH in CMakeLists.txt from SET to +LIST(APPEND...) to allow a setting from the command line to be included. + +23. Updated to Unicode 13.0.0. + +24. CMake build now checks for secure_getenv() and strerror(). Patch by Carlo. + +25. Avoid using [-1] as a suffix in pcre2test because it can provoke a compiler +warning. + +26. Added tests for __attribute__((uninitialized)) to both the configure and +CMake build files, and then applied this attribute to the variable called +stack_frames_vector[] in pcre2_match(). When implemented, this disables +automatic initialization (a facility in clang), which can take time on big +variables. + +27. Updated CMakeLists.txt (patches by Uwe Korn) to add support for +pcre2-config, the libpcre*.pc files, SOVERSION, VERSION and the +MACHO_*_VERSIONS settings for CMake builds. + +28. Another patch to CMakeLists.txt to check for mkostemp (configure already +does). Patch by Carlo Marcelo Arenas Belon. + +29. Check for the existence of memfd_create in both CMake and configure +configurations. Patch by Carlo Marcelo Arenas Belon. + +30. Restrict the configuration setting for the SELinux compatible execmem +allocator (change 10.30/44) to Linux and NetBSD. + + +Version 10.34 21-November-2019 +------------------------------ + +1. The maximum number of capturing subpatterns is 65535 (documented), but no +check on this was ever implemented. This omission has been rectified; it fixes +ClusterFuzz 14376. + +2. Improved the invalid utf32 support of the JIT compiler. Now it correctly +detects invalid characters in the 0xd800-0xdfff range. + +3. Fix minor typo bug in JIT compile when \X is used in a non-UTF string. + +4. Add support for matching in invalid UTF strings to the pcre2_match() +interpreter, and integrate with the existing JIT support via the new +PCRE2_MATCH_INVALID_UTF compile-time option. + +5. Give more error detail for invalid UTF-8 when detected in pcre2grep. + +6. Add support for invalid UTF-8 to pcre2grep. + +7. Adjust the limit for "must have" code unit searching, in particular, +increase it substantially for non-anchored patterns. + +8. Allow (*ACCEPT) to be quantified, because an ungreedy quantifier with a zero +minimum is potentially useful. + +9. Some changes to the way the minimum subject length is handled: + + * When PCRE2_NO_START_OPTIMIZE is set, no minimum length is computed; + pcre2test now omits this item instead of showing a value of zero. + + * An incorrect minimum length could be calculated for a pattern that + contained (*ACCEPT) inside a qualified group whose minimum repetition was + zero, for example /A(?:(*ACCEPT))?B/, which incorrectly computed a minimum + of 2. The minimum length scan no longer happens for a pattern that + contains (*ACCEPT). + + * When no minimum length is set by the normal scan, but a first and/or last + code unit is recorded, set the minimum to 1 or 2 as appropriate. + + * When a pattern contains multiple groups with the same number, a back + reference cannot know which one to scan for a minimum length. This used to + cause the minimum length finder to give up with no result. Now it treats + such references as not adding to the minimum length (which it should have + done all along). + + * Furthermore, the above action now happens only if the back reference is to + a group that exists more than once in a pattern instead of any back + reference in a pattern with duplicate numbers. + +10. A (*MARK) value inside a successful condition was not being returned by the +interpretive matcher (it was returned by JIT). This bug has been mended. + +11. A bug in pcre2grep meant that -o without an argument (or -o0) didn't work +if the pattern had more than 32 capturing parentheses. This is fixed. In +addition (a) the default limit for groups requested by -o<n> has been raised to +50, (b) the new --om-capture option changes the limit, (c) an error is raised +if -o asks for a group that is above the limit. + +12. The quantifier {1} was always being ignored, but this is incorrect when it +is made possessive and applied to an item in parentheses, because a +parenthesized item may contain multiple branches or other backtracking points, +for example /(a|ab){1}+c/ or /(a+){1}+a/. + +13. For partial matches, pcre2test was always showing the maximum lookbehind +characters, flagged with "<", which is misleading when the lookbehind didn't +actually look behind the start (because it was later in the pattern). Showing +all consulted preceding characters for partial matches is now controlled by the +existing "allusedtext" modifier and, as for complete matches, this facility is +available only for non-JIT matching, because JIT does not maintain the first +and last consulted characters. + +14. DFA matching (using pcre2_dfa_match()) was not recognising a partial match +if the end of the subject was encountered in a lookahead (conditional or +otherwise), an atomic group, or a recursion. + +15. Give error if pcre2test -t, -T, -tm or -TM is given an argument of zero. + +16. Check for integer overflow when computing lookbehind lengths. Fixes +Clusterfuzz issue 15636. + +17. Implemented non-atomic positive lookaround assertions. + +18. If a lookbehind contained a lookahead that contained another lookbehind +within it, the nested lookbehind was not correctly processed. For example, if +/(?<=(?=(?<=a)))b/ was matched to "ab" it gave no match instead of matching +"b". + +19. Implemented pcre2_get_match_data_size(). + +20. Two alterations to partial matching: + + (a) The definition of a partial match is slightly changed: if a pattern + contains any lookbehinds, an empty partial match may be given, because this + is another situation where adding characters to the current subject can + lead to a full match. Example: /c*+(?<=[bc])/ with subject "ab". + + (b) Similarly, if a pattern could match an empty string, an empty partial + match may be given. Example: /(?![ab]).*/ with subject "ab". This case + applies only to PCRE2_PARTIAL_HARD. + + (c) An empty string partial hard match can be returned for \z and \Z as it + is documented that they shouldn't match. + +21. A branch that started with (*ACCEPT) was not being recognized as one that +could match an empty string. + +22. Corrected pcre2_set_character_tables() tables data type: was const unsigned +char * instead of const uint8_t *, as generated by pcre2_maketables(). + +23. Upgraded to Unicode 12.1.0. + +24. Add -jitfast command line option to pcre2test (to make all the jit options +available directly). + +25. Make pcre2test -C show if libreadline or libedit is supported. + +26. If the length of one branch of a group exceeded 65535 (the maximum value +that is remembered as a minimum length), the whole group's length was +incorrectly recorded as 65535, leading to incorrect "no match" when start-up +optimizations were in force. + +27. The "rightmost consulted character" value was not always correct; in +particular, if a pattern ended with a negative lookahead, characters that were +inspected in that lookahead were not included. + +28. Add the pcre2_maketables_free() function. + +29. The start-up optimization that looks for a unique initial matching +code unit in the interpretive engines uses memchr() in 8-bit mode. When the +search is caseless, it was doing so inefficiently, which ended up slowing down +the match drastically when the subject was very long. The revised code (a) +remembers if one case is not found, so it never repeats the search for that +case after a bumpalong and (b) when one case has been found, it searches only +up to that position for an earlier occurrence of the other case. This fix +applies to both interpretive pcre2_match() and to pcre2_dfa_match(). + +30. While scanning to find the minimum length of a group, if any branch has +minimum length zero, there is no need to scan any subsequent branches (a small +compile-time performance improvement). + +31. Installed a .gitignore file on a user's suggestion. When using the svn +repository with git (through git svn) this helps keep it tidy. + +32. Add underflow check in JIT which may occur when the value of subject +string pointer is close to 0. + +33. Arrange for classes such as [Aa] which contain just the two cases of the +same character, to be treated as a single caseless character. This causes the +first and required code unit optimizations to kick in where relevant. + +34. Improve the bitmap of starting bytes for positive classes that include wide +characters, but no property types, in UTF-8 mode. Previously, on encountering +such a class, the bits for all bytes greater than \xc4 were set, thus +specifying any character with codepoint >= 0x100. Now the only bits that are +set are for the relevant bytes that start the wide characters. This can give a +noticeable performance improvement. + +35. If the bitmap of starting code units contains only 1 or 2 bits, replace it +with a single starting code unit (1 bit) or a caseless single starting code +unit if the two relevant characters are case-partners. This is particularly +relevant to the 8-bit library, though it applies to all. It can give a +performance boost for patterns such as [Ww]ord and (word|WORD). However, this +optimization doesn't happen if there is a "required" code unit of the same +value (because the search for a "required" code unit starts at the match start +for non-unique first code unit patterns, but after a unique first code unit, +and patterns such as a*a need the former action). + +36. Small patch to pcre2posix.c to set the erroroffset field to -1 immediately +after a successful compile, instead of at the start of matching to avoid a +sanitizer complaint (regexec is supposed to be thread safe). + +37. Add NEON vectorization to JIT to speed up matching of first character and +pairs of characters on ARM64 CPUs. + +38. If a non-ASCII character was the first in a starting assertion in a +caseless match, the "first code unit" optimization did not get the casing +right, and the assertion failed to match a character in the other case if it +did not start with the same code unit. + +39. Fixed the incorrect computation of jump sizes on x86 CPUs in JIT. A masking +operation was incorrectly removed in r1136. Reported by Ralf Junker. + Version 10.33 16-April-2019 --------------------------- diff --git a/dist2/CheckMan b/dist2/CheckMan index 56867465..2f84f99b 100755 --- a/dist2/CheckMan +++ b/dist2/CheckMan @@ -16,6 +16,7 @@ while (scalar(@ARGV) > 0) while (<IN>) { + $count = 0; $line++; if (/^\s*$/) { @@ -50,14 +51,24 @@ while (scalar(@ARGV) > 0) $yield = 1; } } - else + elsif (/\\[^ef]|\\f[^IBP]/) { - if (/\\[^ef]|\\f[^IBP]/) - { - printf "Bad backslash in line $line of $file\n"; - $yield = 1; - } - } + printf "Bad backslash in line $line of $file\n"; + $yield = 1; + } + while (/\\f[BI]/g) + { + $count++; + } + while (/\\fP/g) + { + $count--; + } + if ($count != 0) + { + printf "Mismatching formatting in line $line of $file\n"; + $yield = 1; + } } close(IN); diff --git a/dist2/HACKING b/dist2/HACKING index f99616ac..20faf8f4 100644 --- a/dist2/HACKING +++ b/dist2/HACKING @@ -195,6 +195,7 @@ META_END End of pattern (this value is 0x80000000) META_FAIL (*FAIL) META_KET ) closing parenthesis META_LOOKAHEAD (?= start of lookahead +META_LOOKAHEAD_NA (*napla: start of non-atomic lookahead META_LOOKAHEADNOT (?! start of negative lookahead META_NOCAPTURE (?: no capture parens META_PLUS + @@ -286,8 +287,9 @@ The following are also followed just by an offset, but also the lower 16 bits of the main word contain the length of the first branch of the lookbehind group; this is used when generating OP_REVERSE for that branch. -META_LOOKBEHIND (?<= -META_LOOKBEHINDNOT (?<! +META_LOOKBEHIND (?<= start of lookbehind +META_LOOKBEHIND_NA (*naplb: start of non-atomic lookbehind +META_LOOKBEHINDNOT (?<! start of negative lookbehind The following are followed by two elements, the minimum and maximum. Repeat values are limited to 65535 (MAX_REPEAT). A maximum value of "unlimited" is @@ -715,13 +717,15 @@ Assertions ---------- Forward assertions are also just like other subpatterns, but starting with one -of the opcodes OP_ASSERT or OP_ASSERT_NOT. Backward assertions use the opcodes -OP_ASSERTBACK and OP_ASSERTBACK_NOT, and the first opcode inside the assertion -is OP_REVERSE, followed by a count of the number of characters to move back the -pointer in the subject string. In ASCII or UTF-32 mode, the count is also the -number of code units, but in UTF-8/16 mode each character may occupy more than -one code unit. A separate count is present in each alternative of a lookbehind -assertion, allowing them to have different (but fixed) lengths. +of the opcodes OP_ASSERT, OP_ASSERT_NA (non-atomic assertion), or +OP_ASSERT_NOT. Backward assertions use the opcodes OP_ASSERTBACK, +OP_ASSERTBACK_NA, and OP_ASSERTBACK_NOT, and the first opcode inside the +assertion is OP_REVERSE, followed by a count of the number of characters to +move back the pointer in the subject string. In ASCII or UTF-32 mode, the count +is also the number of code units, but in UTF-8/16 mode each character may +occupy more than one code unit. A separate count is present in each alternative +of a lookbehind assertion, allowing each branch to have a different (but fixed) +length. Conditional subpatterns @@ -754,11 +758,11 @@ tests the PCRE2 version number. This compiles into one of the opcodes OP_TRUE or OP_FALSE. If a condition is not a back reference, recursion test, DEFINE, or VERSION, it -must start with a parenthesized assertion, whose opcode normally immediately -follows OP_COND or OP_SCOND. However, if automatic callouts are enabled, a -callout is inserted immediately before the assertion. It is also possible to -insert a manual callout at this point. Only assertion conditions may have -callouts preceding the condition. +must start with a parenthesized atomic assertion, whose opcode normally +immediately follows OP_COND or OP_SCOND. However, if automatic callouts are +enabled, a callout is inserted immediately before the assertion. It is also +possible to insert a manual callout at this point. Only assertion conditions +may have callouts preceding the condition. A condition that is the negative assertion (?!) is optimized to OP_FAIL in all parts of the pattern, so this is another opcode that may appear as a condition. @@ -823,4 +827,4 @@ not a real opcode, but is used to check at compile time that tables indexed by opcode are the correct length, in order to catch updating errors. Philip Hazel -20 July 2018 +12 July 2019 diff --git a/dist2/LICENCE b/dist2/LICENCE index 142b3b3f..155d0731 100644 --- a/dist2/LICENCE +++ b/dist2/LICENCE @@ -20,13 +20,13 @@ THE BASIC LIBRARY FUNCTIONS --------------------------- Written by: Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk +Email local part: Philip.Hazel +Email domain: gmail.com University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2019 University of Cambridge +Copyright (c) 1997-2020 University of Cambridge All rights reserved. @@ -37,7 +37,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Email domain: freemail.hu -Copyright(c) 2010-2019 Zoltan Herczeg +Copyright(c) 2010-2020 Zoltan Herczeg All rights reserved. @@ -48,7 +48,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Email domain: freemail.hu -Copyright(c) 2009-2019 Zoltan Herczeg +Copyright(c) 2009-2020 Zoltan Herczeg All rights reserved. diff --git a/dist2/Makefile.am b/dist2/Makefile.am index b5e3635e..bd8e6f0c 100644 --- a/dist2/Makefile.am +++ b/dist2/Makefile.am @@ -46,6 +46,7 @@ dist_html_DATA = \ doc/html/pcre2_general_context_free.html \ doc/html/pcre2_get_error_message.html \ doc/html/pcre2_get_mark.html \ + doc/html/pcre2_get_match_data_size.html \ doc/html/pcre2_get_ovector_count.html \ doc/html/pcre2_get_ovector_pointer.html \ doc/html/pcre2_get_startchar.html \ @@ -56,6 +57,7 @@ dist_html_DATA = \ doc/html/pcre2_jit_stack_create.html \ doc/html/pcre2_jit_stack_free.html \ doc/html/pcre2_maketables.html \ + doc/html/pcre2_maketables_free.html \ doc/html/pcre2_match.html \ doc/html/pcre2_match_context_copy.html \ doc/html/pcre2_match_context_create.html \ @@ -140,6 +142,7 @@ dist_man_MANS = \ doc/pcre2_general_context_free.3 \ doc/pcre2_get_error_message.3 \ doc/pcre2_get_mark.3 \ + doc/pcre2_get_match_data_size.3 \ doc/pcre2_get_ovector_count.3 \ doc/pcre2_get_ovector_pointer.3 \ doc/pcre2_get_startchar.3 \ @@ -150,6 +153,7 @@ dist_man_MANS = \ doc/pcre2_jit_stack_create.3 \ doc/pcre2_jit_stack_free.3 \ doc/pcre2_maketables.3 \ + doc/pcre2_maketables_free.3 \ doc/pcre2_match.3 \ doc/pcre2_match_context_copy.3 \ doc/pcre2_match_context_create.3 \ @@ -321,18 +325,18 @@ include_HEADERS = src/pcre2posix.h bin_SCRIPTS = pcre2-config ## --------------------------------------------------------------- -## The dftables program is used to rebuild character tables before compiling -## PCRE2, if --enable-rebuild-chartables is specified. It is not a user-visible -## program. The default (when --enable-rebuild-chartables is not specified) is -## to copy a distributed set of tables that are defined for ASCII code. In this -## case, dftables is not needed. +## The pcre2_dftables program is used to rebuild character tables before +## compiling PCRE2, if --enable-rebuild-chartables is specified. It is not an +## installed program. The default (when --enable-rebuild-chartables is not +## specified) is to copy a distributed set of tables that are defined for ASCII +## code. In this case, pcre2_dftables is not needed. if WITH_REBUILD_CHARTABLES -noinst_PROGRAMS += dftables -dftables_SOURCES = src/dftables.c -src/pcre2_chartables.c: dftables$(EXEEXT) +noinst_PROGRAMS += pcre2_dftables +pcre2_dftables_SOURCES = src/pcre2_dftables.c +src/pcre2_chartables.c: pcre2_dftables$(EXEEXT) rm -f $@ - ./dftables$(EXEEXT) $@ + ./pcre2_dftables$(EXEEXT) $@ else src/pcre2_chartables.c: $(srcdir)/src/pcre2_chartables.c.dist rm -f $@ @@ -358,6 +362,8 @@ COMMON_SOURCES = \ src/pcre2_internal.h \ src/pcre2_intmodedep.h \ src/pcre2_jit_compile.c \ + src/pcre2_jit_neon_inc.h \ + src/pcre2_jit_simd_inc.h \ src/pcre2_maketables.c \ src/pcre2_match.c \ src/pcre2_match_data.c \ @@ -385,6 +391,7 @@ nodist_libpcre2_8_la_SOURCES = \ libpcre2_8_la_CFLAGS = \ -DPCRE2_CODE_UNIT_WIDTH=8 \ $(VISIBILITY_CFLAGS) \ + $(CET_CFLAGS) \ $(AM_CFLAGS) libpcre2_8_la_LIBADD = endif # WITH_PCRE2_8 @@ -398,6 +405,7 @@ nodist_libpcre2_16_la_SOURCES = \ libpcre2_16_la_CFLAGS = \ -DPCRE2_CODE_UNIT_WIDTH=16 \ $(VISIBILITY_CFLAGS) \ + $(CET_CFLAGS) \ $(AM_CFLAGS) libpcre2_16_la_LIBADD = endif # WITH_PCRE2_16 @@ -411,6 +419,7 @@ nodist_libpcre2_32_la_SOURCES = \ libpcre2_32_la_CFLAGS = \ -DPCRE2_CODE_UNIT_WIDTH=32 \ $(VISIBILITY_CFLAGS) \ + $(CET_CFLAGS) \ $(AM_CFLAGS) libpcre2_32_la_LIBADD = endif # WITH_PCRE2_32 @@ -439,15 +448,15 @@ EXTRA_DIST += \ src/sljit/sljitNativePPC_32.c \ src/sljit/sljitNativePPC_64.c \ src/sljit/sljitNativePPC_common.c \ + src/sljit/sljitNativeS390X.c \ src/sljit/sljitNativeSPARC_32.c \ src/sljit/sljitNativeSPARC_common.c \ - src/sljit/sljitNativeTILEGX-encoder.c \ - src/sljit/sljitNativeTILEGX_64.c \ src/sljit/sljitNativeX86_32.c \ src/sljit/sljitNativeX86_64.c \ src/sljit/sljitNativeX86_common.c \ src/sljit/sljitProtExecAllocator.c \ - src/sljit/sljitUtils.c + src/sljit/sljitUtils.c \ + src/sljit/sljitWXExecAllocator.c # Some of the JIT sources are also in separate files that are #included. @@ -628,6 +637,7 @@ EXTRA_DIST += \ testdata/grepoutputCN \ testdata/grepoutputN \ testdata/greppatN4 \ + testdata/testbtables \ testdata/testinput1 \ testdata/testinput2 \ testdata/testinput3 \ diff --git a/dist2/Makefile.in b/dist2/Makefile.in index 2944b0e3..350ef054 100644 --- a/dist2/Makefile.in +++ b/dist2/Makefile.in @@ -1,7 +1,7 @@ -# Makefile.in generated by automake 1.16.1 from Makefile.am. +# Makefile.in generated by automake 1.16.2 from Makefile.am. # @configure_input@ -# Copyright (C) 1994-2018 Free Software Foundation, Inc. +# Copyright (C) 1994-2020 Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -96,7 +96,7 @@ host_triplet = @host@ TESTS = $(am__EXEEXT_4) RunTest $(am__append_32) bin_PROGRAMS = $(am__EXEEXT_1) pcre2test$(EXEEXT) noinst_PROGRAMS = $(am__EXEEXT_2) $(am__EXEEXT_3) $(am__EXEEXT_4) -@WITH_REBUILD_CHARTABLES_TRUE@am__append_1 = dftables +@WITH_REBUILD_CHARTABLES_TRUE@am__append_1 = pcre2_dftables @WITH_PCRE2_8_TRUE@am__append_2 = libpcre2-8.la @WITH_PCRE2_16_TRUE@am__append_3 = libpcre2-16.la @WITH_PCRE2_32_TRUE@am__append_4 = libpcre2-32.la @@ -161,7 +161,7 @@ am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(libdir)" \ "$(DESTDIR)$(man3dir)" "$(DESTDIR)$(docdir)" \ "$(DESTDIR)$(htmldir)" "$(DESTDIR)$(pkgconfigdir)" \ "$(DESTDIR)$(includedir)" "$(DESTDIR)$(includedir)" -@WITH_REBUILD_CHARTABLES_TRUE@am__EXEEXT_2 = dftables$(EXEEXT) +@WITH_REBUILD_CHARTABLES_TRUE@am__EXEEXT_2 = pcre2_dftables$(EXEEXT) @WITH_FUZZ_SUPPORT_TRUE@@WITH_PCRE2_8_TRUE@am__EXEEXT_3 = pcre2fuzzcheck$(EXEEXT) @WITH_JIT_TRUE@am__EXEEXT_4 = pcre2_jit_test$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) @@ -213,7 +213,8 @@ am__libpcre2_16_la_SOURCES_DIST = src/pcre2_auto_possess.c \ src/pcre2_convert.c src/pcre2_dfa_match.c src/pcre2_error.c \ src/pcre2_extuni.c src/pcre2_find_bracket.c \ src/pcre2_internal.h src/pcre2_intmodedep.h \ - src/pcre2_jit_compile.c src/pcre2_maketables.c \ + src/pcre2_jit_compile.c src/pcre2_jit_neon_inc.h \ + src/pcre2_jit_simd_inc.h src/pcre2_maketables.c \ src/pcre2_match.c src/pcre2_match_data.c src/pcre2_newline.c \ src/pcre2_ord2utf.c src/pcre2_pattern_info.c \ src/pcre2_script_run.c src/pcre2_serialize.c \ @@ -267,7 +268,8 @@ am__libpcre2_32_la_SOURCES_DIST = src/pcre2_auto_possess.c \ src/pcre2_convert.c src/pcre2_dfa_match.c src/pcre2_error.c \ src/pcre2_extuni.c src/pcre2_find_bracket.c \ src/pcre2_internal.h src/pcre2_intmodedep.h \ - src/pcre2_jit_compile.c src/pcre2_maketables.c \ + src/pcre2_jit_compile.c src/pcre2_jit_neon_inc.h \ + src/pcre2_jit_simd_inc.h src/pcre2_maketables.c \ src/pcre2_match.c src/pcre2_match_data.c src/pcre2_newline.c \ src/pcre2_ord2utf.c src/pcre2_pattern_info.c \ src/pcre2_script_run.c src/pcre2_serialize.c \ @@ -317,7 +319,8 @@ am__libpcre2_8_la_SOURCES_DIST = src/pcre2_auto_possess.c \ src/pcre2_convert.c src/pcre2_dfa_match.c src/pcre2_error.c \ src/pcre2_extuni.c src/pcre2_find_bracket.c \ src/pcre2_internal.h src/pcre2_intmodedep.h \ - src/pcre2_jit_compile.c src/pcre2_maketables.c \ + src/pcre2_jit_compile.c src/pcre2_jit_neon_inc.h \ + src/pcre2_jit_simd_inc.h src/pcre2_maketables.c \ src/pcre2_match.c src/pcre2_match_data.c src/pcre2_newline.c \ src/pcre2_ord2utf.c src/pcre2_pattern_info.c \ src/pcre2_script_run.c src/pcre2_serialize.c \ @@ -370,11 +373,11 @@ libpcre2_posix_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ $(libpcre2_posix_la_CFLAGS) $(CFLAGS) \ $(libpcre2_posix_la_LDFLAGS) $(LDFLAGS) -o $@ @WITH_PCRE2_8_TRUE@am_libpcre2_posix_la_rpath = -rpath $(libdir) -am__dftables_SOURCES_DIST = src/dftables.c -@WITH_REBUILD_CHARTABLES_TRUE@am_dftables_OBJECTS = \ -@WITH_REBUILD_CHARTABLES_TRUE@ src/dftables.$(OBJEXT) -dftables_OBJECTS = $(am_dftables_OBJECTS) -dftables_LDADD = $(LDADD) +am__pcre2_dftables_SOURCES_DIST = src/pcre2_dftables.c +@WITH_REBUILD_CHARTABLES_TRUE@am_pcre2_dftables_OBJECTS = \ +@WITH_REBUILD_CHARTABLES_TRUE@ src/pcre2_dftables.$(OBJEXT) +pcre2_dftables_OBJECTS = $(am_pcre2_dftables_OBJECTS) +pcre2_dftables_LDADD = $(LDADD) am__pcre2_jit_test_SOURCES_DIST = src/pcre2_jit_test.c @WITH_JIT_TRUE@am_pcre2_jit_test_OBJECTS = \ @WITH_JIT_TRUE@ src/pcre2_jit_test-pcre2_jit_test.$(OBJEXT) @@ -438,7 +441,6 @@ DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/src depcomp = $(SHELL) $(top_srcdir)/depcomp am__maybe_remake_depfiles = depfiles am__depfiles_remade = src/$(DEPDIR)/_libs_libpcre2_fuzzsupport_a-pcre2_fuzzsupport.Po \ - src/$(DEPDIR)/dftables.Po \ src/$(DEPDIR)/libpcre2_16_la-pcre2_auto_possess.Plo \ src/$(DEPDIR)/libpcre2_16_la-pcre2_chartables.Plo \ src/$(DEPDIR)/libpcre2_16_la-pcre2_compile.Plo \ @@ -521,6 +523,7 @@ am__depfiles_remade = src/$(DEPDIR)/_libs_libpcre2_fuzzsupport_a-pcre2_fuzzsuppo src/$(DEPDIR)/libpcre2_8_la-pcre2_valid_utf.Plo \ src/$(DEPDIR)/libpcre2_8_la-pcre2_xclass.Plo \ src/$(DEPDIR)/libpcre2_posix_la-pcre2posix.Plo \ + src/$(DEPDIR)/pcre2_dftables.Po \ src/$(DEPDIR)/pcre2_jit_test-pcre2_jit_test.Po \ src/$(DEPDIR)/pcre2fuzzcheck-pcre2_fuzzsupport.Po \ src/$(DEPDIR)/pcre2grep-pcre2grep.Po \ @@ -548,7 +551,7 @@ SOURCES = $(_libs_libpcre2_fuzzsupport_a_SOURCES) \ $(libpcre2_16_la_SOURCES) $(nodist_libpcre2_16_la_SOURCES) \ $(libpcre2_32_la_SOURCES) $(nodist_libpcre2_32_la_SOURCES) \ $(libpcre2_8_la_SOURCES) $(nodist_libpcre2_8_la_SOURCES) \ - $(libpcre2_posix_la_SOURCES) $(dftables_SOURCES) \ + $(libpcre2_posix_la_SOURCES) $(pcre2_dftables_SOURCES) \ $(pcre2_jit_test_SOURCES) $(pcre2fuzzcheck_SOURCES) \ $(pcre2grep_SOURCES) $(pcre2test_SOURCES) DIST_SOURCES = $(am___libs_libpcre2_fuzzsupport_a_SOURCES_DIST) \ @@ -556,7 +559,7 @@ DIST_SOURCES = $(am___libs_libpcre2_fuzzsupport_a_SOURCES_DIST) \ $(am__libpcre2_32_la_SOURCES_DIST) \ $(am__libpcre2_8_la_SOURCES_DIST) \ $(am__libpcre2_posix_la_SOURCES_DIST) \ - $(am__dftables_SOURCES_DIST) \ + $(am__pcre2_dftables_SOURCES_DIST) \ $(am__pcre2_jit_test_SOURCES_DIST) \ $(am__pcre2fuzzcheck_SOURCES_DIST) \ $(am__pcre2grep_SOURCES_DIST) $(pcre2test_SOURCES) @@ -803,6 +806,7 @@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ CC = @CC@ CCDEPMODE = @CCDEPMODE@ +CET_CFLAGS = @CET_CFLAGS@ CFLAGS = @CFLAGS@ CPP = @CPP@ CPPFLAGS = @CPPFLAGS@ @@ -843,6 +847,7 @@ LIBREADLINE = @LIBREADLINE@ LIBS = @LIBS@ LIBTOOL = @LIBTOOL@ LIBZ = @LIBZ@ +LIB_POSTFIX = @LIB_POSTFIX@ LIPO = @LIPO@ LN_S = @LN_S@ LTLIBOBJS = @LTLIBOBJS@ @@ -982,6 +987,7 @@ dist_html_DATA = \ doc/html/pcre2_general_context_free.html \ doc/html/pcre2_get_error_message.html \ doc/html/pcre2_get_mark.html \ + doc/html/pcre2_get_match_data_size.html \ doc/html/pcre2_get_ovector_count.html \ doc/html/pcre2_get_ovector_pointer.html \ doc/html/pcre2_get_startchar.html \ @@ -992,6 +998,7 @@ dist_html_DATA = \ doc/html/pcre2_jit_stack_create.html \ doc/html/pcre2_jit_stack_free.html \ doc/html/pcre2_maketables.html \ + doc/html/pcre2_maketables_free.html \ doc/html/pcre2_match.html \ doc/html/pcre2_match_context_copy.html \ doc/html/pcre2_match_context_create.html \ @@ -1076,6 +1083,7 @@ dist_man_MANS = \ doc/pcre2_general_context_free.3 \ doc/pcre2_get_error_message.3 \ doc/pcre2_get_mark.3 \ + doc/pcre2_get_match_data_size.3 \ doc/pcre2_get_ovector_count.3 \ doc/pcre2_get_ovector_pointer.3 \ doc/pcre2_get_startchar.3 \ @@ -1086,6 +1094,7 @@ dist_man_MANS = \ doc/pcre2_jit_stack_create.3 \ doc/pcre2_jit_stack_free.3 \ doc/pcre2_maketables.3 \ + doc/pcre2_maketables_free.3 \ doc/pcre2_match.3 \ doc/pcre2_match_context_copy.3 \ doc/pcre2_match_context_create.3 \ @@ -1201,21 +1210,20 @@ EXTRA_DIST = m4/ax_pthread.m4 m4/pcre2_visibility.m4 \ src/sljit/sljitNativeMIPS_32.c src/sljit/sljitNativeMIPS_64.c \ src/sljit/sljitNativeMIPS_common.c \ src/sljit/sljitNativePPC_32.c src/sljit/sljitNativePPC_64.c \ - src/sljit/sljitNativePPC_common.c \ + src/sljit/sljitNativePPC_common.c src/sljit/sljitNativeS390X.c \ src/sljit/sljitNativeSPARC_32.c \ src/sljit/sljitNativeSPARC_common.c \ - src/sljit/sljitNativeTILEGX-encoder.c \ - src/sljit/sljitNativeTILEGX_64.c src/sljit/sljitNativeX86_32.c \ - src/sljit/sljitNativeX86_64.c \ + src/sljit/sljitNativeX86_32.c src/sljit/sljitNativeX86_64.c \ src/sljit/sljitNativeX86_common.c \ src/sljit/sljitProtExecAllocator.c src/sljit/sljitUtils.c \ - src/pcre2_jit_match.c src/pcre2_jit_misc.c \ - src/pcre2_printint.c RunTest.bat $(am__append_33) \ - testdata/grepbinary testdata/grepfilelist testdata/grepinput \ - testdata/grepinput3 testdata/grepinput8 testdata/grepinputM \ - testdata/grepinputv testdata/grepinputx testdata/greplist \ - testdata/grepoutput testdata/grepoutput8 testdata/grepoutputC \ - testdata/grepoutputCN testdata/grepoutputN testdata/greppatN4 \ + src/sljit/sljitWXExecAllocator.c src/pcre2_jit_match.c \ + src/pcre2_jit_misc.c src/pcre2_printint.c RunTest.bat \ + $(am__append_33) testdata/grepbinary testdata/grepfilelist \ + testdata/grepinput testdata/grepinput3 testdata/grepinput8 \ + testdata/grepinputM testdata/grepinputv testdata/grepinputx \ + testdata/greplist testdata/grepoutput testdata/grepoutput8 \ + testdata/grepoutputC testdata/grepoutputCN \ + testdata/grepoutputN testdata/greppatN4 testdata/testbtables \ testdata/testinput1 testdata/testinput2 testdata/testinput3 \ testdata/testinput4 testdata/testinput5 testdata/testinput6 \ testdata/testinput7 testdata/testinput8 testdata/testinput9 \ @@ -1259,7 +1267,7 @@ include_HEADERS = src/pcre2posix.h # This is the "config" script. bin_SCRIPTS = pcre2-config -@WITH_REBUILD_CHARTABLES_TRUE@dftables_SOURCES = src/dftables.c +@WITH_REBUILD_CHARTABLES_TRUE@pcre2_dftables_SOURCES = src/pcre2_dftables.c BUILT_SOURCES = src/pcre2_chartables.c NODIST_SOURCES = src/pcre2_chartables.c COMMON_SOURCES = \ @@ -1275,6 +1283,8 @@ COMMON_SOURCES = \ src/pcre2_internal.h \ src/pcre2_intmodedep.h \ src/pcre2_jit_compile.c \ + src/pcre2_jit_neon_inc.h \ + src/pcre2_jit_simd_inc.h \ src/pcre2_maketables.c \ src/pcre2_match.c \ src/pcre2_match_data.c \ @@ -1300,8 +1310,9 @@ COMMON_SOURCES = \ @WITH_PCRE2_8_TRUE@ $(NODIST_SOURCES) @WITH_PCRE2_8_TRUE@libpcre2_8_la_CFLAGS = -DPCRE2_CODE_UNIT_WIDTH=8 \ -@WITH_PCRE2_8_TRUE@ $(VISIBILITY_CFLAGS) $(AM_CFLAGS) \ -@WITH_PCRE2_8_TRUE@ $(am__append_5) $(am__append_8) +@WITH_PCRE2_8_TRUE@ $(VISIBILITY_CFLAGS) $(CET_CFLAGS) \ +@WITH_PCRE2_8_TRUE@ $(AM_CFLAGS) $(am__append_5) \ +@WITH_PCRE2_8_TRUE@ $(am__append_8) @WITH_PCRE2_8_TRUE@libpcre2_8_la_LIBADD = @WITH_PCRE2_16_TRUE@libpcre2_16_la_SOURCES = \ @WITH_PCRE2_16_TRUE@ $(COMMON_SOURCES) @@ -1311,8 +1322,9 @@ COMMON_SOURCES = \ @WITH_PCRE2_16_TRUE@libpcre2_16_la_CFLAGS = \ @WITH_PCRE2_16_TRUE@ -DPCRE2_CODE_UNIT_WIDTH=16 \ -@WITH_PCRE2_16_TRUE@ $(VISIBILITY_CFLAGS) $(AM_CFLAGS) \ -@WITH_PCRE2_16_TRUE@ $(am__append_6) $(am__append_9) +@WITH_PCRE2_16_TRUE@ $(VISIBILITY_CFLAGS) $(CET_CFLAGS) \ +@WITH_PCRE2_16_TRUE@ $(AM_CFLAGS) $(am__append_6) \ +@WITH_PCRE2_16_TRUE@ $(am__append_9) @WITH_PCRE2_16_TRUE@libpcre2_16_la_LIBADD = @WITH_PCRE2_32_TRUE@libpcre2_32_la_SOURCES = \ @WITH_PCRE2_32_TRUE@ $(COMMON_SOURCES) @@ -1322,8 +1334,9 @@ COMMON_SOURCES = \ @WITH_PCRE2_32_TRUE@libpcre2_32_la_CFLAGS = \ @WITH_PCRE2_32_TRUE@ -DPCRE2_CODE_UNIT_WIDTH=32 \ -@WITH_PCRE2_32_TRUE@ $(VISIBILITY_CFLAGS) $(AM_CFLAGS) \ -@WITH_PCRE2_32_TRUE@ $(am__append_7) $(am__append_10) +@WITH_PCRE2_32_TRUE@ $(VISIBILITY_CFLAGS) $(CET_CFLAGS) \ +@WITH_PCRE2_32_TRUE@ $(AM_CFLAGS) $(am__append_7) \ +@WITH_PCRE2_32_TRUE@ $(am__append_10) @WITH_PCRE2_32_TRUE@libpcre2_32_la_LIBADD = @WITH_PCRE2_8_TRUE@libpcre2_8_la_LDFLAGS = $(EXTRA_LIBPCRE2_8_LDFLAGS) @WITH_PCRE2_16_TRUE@libpcre2_16_la_LDFLAGS = $(EXTRA_LIBPCRE2_16_LDFLAGS) @@ -1741,12 +1754,12 @@ src/libpcre2_posix_la-pcre2posix.lo: src/$(am__dirstamp) \ libpcre2-posix.la: $(libpcre2_posix_la_OBJECTS) $(libpcre2_posix_la_DEPENDENCIES) $(EXTRA_libpcre2_posix_la_DEPENDENCIES) $(AM_V_CCLD)$(libpcre2_posix_la_LINK) $(am_libpcre2_posix_la_rpath) $(libpcre2_posix_la_OBJECTS) $(libpcre2_posix_la_LIBADD) $(LIBS) -src/dftables.$(OBJEXT): src/$(am__dirstamp) \ +src/pcre2_dftables.$(OBJEXT): src/$(am__dirstamp) \ src/$(DEPDIR)/$(am__dirstamp) -dftables$(EXEEXT): $(dftables_OBJECTS) $(dftables_DEPENDENCIES) $(EXTRA_dftables_DEPENDENCIES) - @rm -f dftables$(EXEEXT) - $(AM_V_CCLD)$(LINK) $(dftables_OBJECTS) $(dftables_LDADD) $(LIBS) +pcre2_dftables$(EXEEXT): $(pcre2_dftables_OBJECTS) $(pcre2_dftables_DEPENDENCIES) $(EXTRA_pcre2_dftables_DEPENDENCIES) + @rm -f pcre2_dftables$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(pcre2_dftables_OBJECTS) $(pcre2_dftables_LDADD) $(LIBS) src/pcre2_jit_test-pcre2_jit_test.$(OBJEXT): src/$(am__dirstamp) \ src/$(DEPDIR)/$(am__dirstamp) @@ -1816,7 +1829,6 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/_libs_libpcre2_fuzzsupport_a-pcre2_fuzzsupport.Po@am__quote@ # am--include-marker -@AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/dftables.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_16_la-pcre2_auto_possess.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_16_la-pcre2_chartables.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_16_la-pcre2_compile.Plo@am__quote@ # am--include-marker @@ -1899,6 +1911,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_8_la-pcre2_valid_utf.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_8_la-pcre2_xclass.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/libpcre2_posix_la-pcre2posix.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/pcre2_dftables.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/pcre2_jit_test-pcre2_jit_test.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/pcre2fuzzcheck-pcre2_fuzzsupport.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@src/$(DEPDIR)/pcre2grep-pcre2grep.Po@am__quote@ # am--include-marker @@ -3072,6 +3085,10 @@ dist-xz: distdir tardir=$(distdir) && $(am__tar) | XZ_OPT=$${XZ_OPT--e} xz -c >$(distdir).tar.xz $(am__post_remove_distdir) +dist-zstd: distdir + tardir=$(distdir) && $(am__tar) | zstd -c $${ZSTD_CLEVEL-$${ZSTD_OPT--19}} >$(distdir).tar.zst + $(am__post_remove_distdir) + dist-tarZ: distdir @echo WARNING: "Support for distribution archives compressed with" \ "legacy program 'compress' is deprecated." >&2 @@ -3113,6 +3130,8 @@ distcheck: dist eval GZIP= gzip $(GZIP_ENV) -dc $(distdir).shar.gz | unshar ;;\ *.zip*) \ unzip $(distdir).zip ;;\ + *.tar.zst*) \ + zstd -dc $(distdir).tar.zst | $(am__untar) ;;\ esac chmod -R a-w $(distdir) chmod u+w $(distdir) @@ -3243,7 +3262,6 @@ clean-am: clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \ distclean: distclean-am -rm -f $(am__CONFIG_DISTCLEAN_FILES) -rm -f src/$(DEPDIR)/_libs_libpcre2_fuzzsupport_a-pcre2_fuzzsupport.Po - -rm -f src/$(DEPDIR)/dftables.Po -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_auto_possess.Plo -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_chartables.Plo -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_compile.Plo @@ -3326,6 +3344,7 @@ distclean: distclean-am -rm -f src/$(DEPDIR)/libpcre2_8_la-pcre2_valid_utf.Plo -rm -f src/$(DEPDIR)/libpcre2_8_la-pcre2_xclass.Plo -rm -f src/$(DEPDIR)/libpcre2_posix_la-pcre2posix.Plo + -rm -f src/$(DEPDIR)/pcre2_dftables.Po -rm -f src/$(DEPDIR)/pcre2_jit_test-pcre2_jit_test.Po -rm -f src/$(DEPDIR)/pcre2fuzzcheck-pcre2_fuzzsupport.Po -rm -f src/$(DEPDIR)/pcre2grep-pcre2grep.Po @@ -3381,7 +3400,6 @@ maintainer-clean: maintainer-clean-am -rm -f $(am__CONFIG_DISTCLEAN_FILES) -rm -rf $(top_srcdir)/autom4te.cache -rm -f src/$(DEPDIR)/_libs_libpcre2_fuzzsupport_a-pcre2_fuzzsupport.Po - -rm -f src/$(DEPDIR)/dftables.Po -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_auto_possess.Plo -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_chartables.Plo -rm -f src/$(DEPDIR)/libpcre2_16_la-pcre2_compile.Plo @@ -3464,6 +3482,7 @@ maintainer-clean: maintainer-clean-am -rm -f src/$(DEPDIR)/libpcre2_8_la-pcre2_valid_utf.Plo -rm -f src/$(DEPDIR)/libpcre2_8_la-pcre2_xclass.Plo -rm -f src/$(DEPDIR)/libpcre2_posix_la-pcre2posix.Plo + -rm -f src/$(DEPDIR)/pcre2_dftables.Po -rm -f src/$(DEPDIR)/pcre2_jit_test-pcre2_jit_test.Po -rm -f src/$(DEPDIR)/pcre2fuzzcheck-pcre2_fuzzsupport.Po -rm -f src/$(DEPDIR)/pcre2grep-pcre2grep.Po @@ -3500,27 +3519,27 @@ uninstall-man: uninstall-man1 uninstall-man3 clean-noinstLIBRARIES clean-noinstPROGRAMS cscope \ cscopelist-am ctags ctags-am dist dist-all dist-bzip2 \ dist-gzip dist-lzip dist-shar dist-tarZ dist-xz dist-zip \ - distcheck distclean distclean-compile distclean-generic \ - distclean-hdr distclean-libtool distclean-local distclean-tags \ - distcleancheck distdir distuninstallcheck dvi dvi-am html \ - html-am info info-am install install-am install-binPROGRAMS \ - install-binSCRIPTS install-data install-data-am \ - install-dist_docDATA install-dist_htmlDATA install-dvi \ - install-dvi-am install-exec install-exec-am install-html \ - install-html-am install-includeHEADERS install-info \ - install-info-am install-libLTLIBRARIES install-man \ - install-man1 install-man3 install-nodist_includeHEADERS \ - install-pdf install-pdf-am install-pkgconfigDATA install-ps \ - install-ps-am install-strip installcheck installcheck-am \ - installdirs maintainer-clean maintainer-clean-generic \ - mostlyclean mostlyclean-compile mostlyclean-generic \ - mostlyclean-libtool pdf pdf-am ps ps-am recheck tags tags-am \ - uninstall uninstall-am uninstall-binPROGRAMS \ - uninstall-binSCRIPTS uninstall-dist_docDATA \ - uninstall-dist_htmlDATA uninstall-includeHEADERS \ - uninstall-libLTLIBRARIES uninstall-man uninstall-man1 \ - uninstall-man3 uninstall-nodist_includeHEADERS \ - uninstall-pkgconfigDATA + dist-zstd distcheck distclean distclean-compile \ + distclean-generic distclean-hdr distclean-libtool \ + distclean-local distclean-tags distcleancheck distdir \ + distuninstallcheck dvi dvi-am html html-am info info-am \ + install install-am install-binPROGRAMS install-binSCRIPTS \ + install-data install-data-am install-dist_docDATA \ + install-dist_htmlDATA install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am \ + install-includeHEADERS install-info install-info-am \ + install-libLTLIBRARIES install-man install-man1 install-man3 \ + install-nodist_includeHEADERS install-pdf install-pdf-am \ + install-pkgconfigDATA install-ps install-ps-am install-strip \ + installcheck installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + recheck tags tags-am uninstall uninstall-am \ + uninstall-binPROGRAMS uninstall-binSCRIPTS \ + uninstall-dist_docDATA uninstall-dist_htmlDATA \ + uninstall-includeHEADERS uninstall-libLTLIBRARIES \ + uninstall-man uninstall-man1 uninstall-man3 \ + uninstall-nodist_includeHEADERS uninstall-pkgconfigDATA .PRECIOUS: Makefile @@ -3562,9 +3581,9 @@ src/config.h.generic: configure.ac -e 'if(/^\s*$$/){print unless $$blank; $$blank=1;} else{print;$$blank=0;}' \ _generic/src/config.h >$@ rm -rf _generic -@WITH_REBUILD_CHARTABLES_TRUE@src/pcre2_chartables.c: dftables$(EXEEXT) +@WITH_REBUILD_CHARTABLES_TRUE@src/pcre2_chartables.c: pcre2_dftables$(EXEEXT) @WITH_REBUILD_CHARTABLES_TRUE@ rm -f $@ -@WITH_REBUILD_CHARTABLES_TRUE@ ./dftables$(EXEEXT) $@ +@WITH_REBUILD_CHARTABLES_TRUE@ ./pcre2_dftables$(EXEEXT) $@ @WITH_REBUILD_CHARTABLES_FALSE@src/pcre2_chartables.c: $(srcdir)/src/pcre2_chartables.c.dist @WITH_REBUILD_CHARTABLES_FALSE@ rm -f $@ @WITH_REBUILD_CHARTABLES_FALSE@ $(LN_S) $(abs_srcdir)/src/pcre2_chartables.c.dist $(abs_builddir)/src/pcre2_chartables.c @@ -2,8 +2,59 @@ News about PCRE2 releases ------------------------- -Version 10.33-RC1 16-April-2019 -------------------------------- +Version 10.36 04-December-2020 +------------------------------ + +Again, mainly bug fixes and tidies. The only enhancements are the addition of +GNU grep's -m (aka --max-count) option to pcre2grep, and also unifying the +handling of substitution strings for both -O and callouts in pcre2grep, with +the addition of $x{...} and $o{...} to allow for characters whose code points +are greater than 255 in Unicode mode. + +NOTE: there is an outstanding issue with JIT support for MacOS on arm64 +hardware. For details, please see Bugzilla issue #2618. + + +Version 10.35 15-April-2020 +--------------------------- + +Bugfixes, tidies, and a few new enhancements. + +1. Capturing groups that contain recursive backreferences to themselves are no +longer automatically atomic, because the restriction is no longer necessary +as a result of the 10.30 restructuring. + +2. Several new options for pcre2_substitute(). + +3. When Unicode is supported and PCRE2_UCP is set without PCRE2_UTF, Unicode +character properties are used for upper/lower case computations on characters +whose code points are greater than 127. + +4. The character tables (for low-valued characters) can now more easily be +saved and restored in binary. + +5. Updated to Unicode 13.0.0. + + +Version 10.34 21-November-2019 +------------------------------ + +Another release with a few enhancements as well as bugfixes and tidies. The +main new features are: + +1. There is now some support for matching in invalid UTF strings. + +2. Non-atomic positive lookarounds are implemented in the pcre2_match() +interpreter, but not in JIT. + +3. Added two new functions: pcre2_get_match_data_size() and +pcre2_maketables_free(). + +4. Upgraded to Unicode 12.1.0. + + +Version 10.33 16-April-2019 +--------------------------- Yet more bugfixes, tidies, and a few enhancements, summarized here (see ChangeLog for the full list): diff --git a/dist2/NON-AUTOTOOLS-BUILD b/dist2/NON-AUTOTOOLS-BUILD index 39e7620a..a73c058b 100644 --- a/dist2/NON-AUTOTOOLS-BUILD +++ b/dist2/NON-AUTOTOOLS-BUILD @@ -74,14 +74,14 @@ can skip ahead to the CMake section. src/pcre2_chartables.c. OR: - Compile src/dftables.c as a stand-alone program (using -DHAVE_CONFIG_H - if you have set up src/config.h), and then run it with the single - argument "src/pcre2_chartables.c". This generates a set of standard - character tables and writes them to that file. The tables are generated - using the default C locale for your system. If you want to use a locale - that is specified by LC_xxx environment variables, add the -L option to - the dftables command. You must use this method if you are building on a - system that uses EBCDIC code. + Compile src/pcre2_dftables.c as a stand-alone program (using + -DHAVE_CONFIG_H if you have set up src/config.h), and then run it with + the single argument "src/pcre2_chartables.c". This generates a set of + standard character tables and writes them to that file. The tables are + generated using the default C locale for your system. If you want to use + a locale that is specified by LC_xxx environment variables, add the -L + option to the pcre2_dftables command. You must use this method if you + are building on a system that uses EBCDIC code. The tables in src/pcre2_chartables.c are defaults. The caller of PCRE2 can specify alternative tables at run time. diff --git a/dist2/PrepareRelease b/dist2/PrepareRelease index 9aa6b7d5..e7cf8db8 100755 --- a/dist2/PrepareRelease +++ b/dist2/PrepareRelease @@ -190,7 +190,7 @@ files="\ libpcre2-16.pc.in \ libpcre2-32.pc.in \ libpcre2-posix.pc.in \ - src/dftables.c \ + src/pcre2_dftables.c \ src/pcre2.h.in \ src/pcre2_auto_possess.c \ src/pcre2_compile.c \ diff --git a/dist2/README b/dist2/README index ff9a6af6..1d6df8f2 100644 --- a/dist2/README +++ b/dist2/README @@ -5,11 +5,11 @@ PCRE2 is a re-working of the original PCRE1 library to provide an entirely new API. Since its initial release in 2015, there has been further development of the code and it now differs from PCRE1 in more than just the API. There are new features and the internals have been improved. The latest release of PCRE2 is -always available in three alternative formats from: +available in three alternative formats from: - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.tar.gz - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.tar.bz2 - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.zip +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.gz +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.bz2 +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.zip There is a mailing list for discussion about the development of PCRE (both the original and new APIs) at pcre-dev@exim.org. You can access the archives and @@ -164,9 +164,11 @@ library. They are also documented in the pcre2build man page. will be a compile time error. If in doubt, use --enable-jit=auto, which enables JIT only if the current hardware is supported. -. If you are enabling JIT under SELinux you may also want to add - --enable-jit-sealloc, which enables the use of an execmem allocator in JIT - that is compatible with SELinux. This has no effect if JIT is not enabled. +. If you are enabling JIT under SELinux environment you may also want to add + --enable-jit-sealloc, which enables the use of an executable memory allocator + that is compatible with SELinux. Warning: this allocator is experimental! + It does not support fork() operation and may crash when no disk space is + available. This option has no effect if JIT is disabled. . If you do not want to make use of the default support for UTF-8 Unicode character strings in the 8-bit library, UTF-16 Unicode character strings in @@ -267,9 +269,9 @@ library. They are also documented in the pcre2build man page. --enable-rebuild-chartables - a program called dftables is compiled and run in the default C locale when - you obey "make". It builds a source file called pcre2_chartables.c. If you do - not specify this option, pcre2_chartables.c is created as a copy of + a program called pcre2_dftables is compiled and run in the default C locale + when you obey "make". It builds a source file called pcre2_chartables.c. If + you do not specify this option, pcre2_chartables.c is created as a copy of pcre2_chartables.c.dist. See "Character tables" below for further information. @@ -295,8 +297,8 @@ library. They are also documented in the pcre2build man page. unaddressable. This allows it to detect invalid memory accesses, and is mostly useful for debugging PCRE2 itself. -. In environments where the gcc compiler is used and lcov version 1.6 or above - is installed, if you specify +. In environments where the gcc compiler is used and lcov is installed, if you + specify --enable-coverage @@ -546,11 +548,11 @@ Cross-compiling using autotools You can specify CC and CFLAGS in the normal way to the "configure" command, in order to cross-compile PCRE2 for some other host. However, you should NOT -specify --enable-rebuild-chartables, because if you do, the dftables.c source -file is compiled and run on the local host, in order to generate the inbuilt -character tables (the pcre2_chartables.c file). This will probably not work, -because dftables.c needs to be compiled with the local compiler, not the cross -compiler. +specify --enable-rebuild-chartables, because if you do, the pcre2_dftables.c +source file is compiled and run on the local host, in order to generate the +inbuilt character tables (the pcre2_chartables.c file). This will probably not +work, because pcre2_dftables.c needs to be compiled with the local compiler, +not the cross compiler. When --enable-rebuild-chartables is not specified, pcre2_chartables.c is created by making a copy of pcre2_chartables.c.dist, which is a default set of @@ -558,9 +560,10 @@ tables that assumes ASCII code. Cross-compiling with the default tables should not be a problem. If you need to modify the character tables when cross-compiling, you should -move pcre2_chartables.c.dist out of the way, then compile dftables.c by hand -and run it on the local host to make a new version of pcre2_chartables.c.dist. -Then when you cross-compile PCRE2 this new version of the tables will be used. +move pcre2_chartables.c.dist out of the way, then compile pcre2_dftables.c by +hand and run it on the local host to make a new version of +pcre2_chartables.c.dist. See the pcre2build section "Creating character tables +at build time" for more details. Making new tarballs @@ -719,8 +722,8 @@ compile context. The source file called pcre2_chartables.c contains the default set of tables. By default, this is created as a copy of pcre2_chartables.c.dist, which contains tables for ASCII coding. However, if --enable-rebuild-chartables is -specified for ./configure, a different version of pcre2_chartables.c is built -by the program dftables (compiled from dftables.c), which uses the ANSI C +specified for ./configure, a new version of pcre2_chartables.c is built by the +program pcre2_dftables (compiled from pcre2_dftables.c), which uses the ANSI C character handling functions such as isalnum(), isalpha(), isupper(), islower(), etc. to build the table sources. This means that the default C locale that is set for your system will control the contents of these default @@ -730,32 +733,40 @@ file does not get automatically re-generated. The best way to do this is to move pcre2_chartables.c.dist out of the way and replace it with your customized tables. -When the dftables program is run as a result of --enable-rebuild-chartables, -it uses the default C locale that is set on your system. It does not pay -attention to the LC_xxx environment variables. In other words, it uses the -system's default locale rather than whatever the compiling user happens to have -set. If you really do want to build a source set of character tables in a -locale that is specified by the LC_xxx variables, you can run the dftables -program by hand with the -L option. For example: +When the pcre2_dftables program is run as a result of specifying +--enable-rebuild-chartables, it uses the default C locale that is set on your +system. It does not pay attention to the LC_xxx environment variables. In other +words, it uses the system's default locale rather than whatever the compiling +user happens to have set. If you really do want to build a source set of +character tables in a locale that is specified by the LC_xxx variables, you can +run the pcre2_dftables program by hand with the -L option. For example: - ./dftables -L pcre2_chartables.c.special + ./pcre2_dftables -L pcre2_chartables.c.special -The first two 256-byte tables provide lower casing and case flipping functions, -respectively. The next table consists of three 32-byte bit maps which identify -digits, "word" characters, and white space, respectively. These are used when -building 32-byte bit maps that represent character classes for code points less -than 256. The final 256-byte table has bits indicating various character types, -as follows: +The second argument names the file where the source code for the tables is +written. The first two 256-byte tables provide lower casing and case flipping +functions, respectively. The next table consists of a number of 32-byte bit +maps which identify certain character classes such as digits, "word" +characters, white space, etc. These are used when building 32-byte bit maps +that represent character classes for code points less than 256. The final +256-byte table has bits indicating various character types, as follows: 1 white space character 2 letter - 4 decimal digit - 8 hexadecimal digit + 4 lower case letter + 8 decimal digit 16 alphanumeric or '_' - 128 regular expression metacharacter or binary zero -You should not alter the set of characters that contain the 128 bit, as that -will cause PCRE2 to malfunction. +You can also specify -b (with or without -L) when running pcre2_dftables. This +causes the tables to be written in binary instead of as source code. A set of +binary tables can be loaded into memory by an application and passed to +pcre2_compile() in the same way as tables created dynamically by calling +pcre2_maketables(). The tables are just a string of bytes, independent of +hardware characteristics such as endianness. This means they can be bundled +with an application that runs in different environments, to ensure consistent +behaviour. + +See also the pcre2build section "Creating character tables at build time". File manifest @@ -766,7 +777,7 @@ The distribution should contain the files listed below. (A) Source files for the PCRE2 library functions and their headers are found in the src directory: - src/dftables.c auxiliary program for building pcre2_chartables.c + src/pcre2_dftables.c auxiliary program for building pcre2_chartables.c when --enable-rebuild-chartables is specified src/pcre2_chartables.c.dist a default set of character tables that assume @@ -890,6 +901,6 @@ The distribution should contain the files listed below. ) environments Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk -Last updated: 16 April 2019 +Email local part: Philip.Hazel +Email domain: gmail.com +Last updated: 04 December 2020 diff --git a/dist2/RunGrepTest b/dist2/RunGrepTest index bac1f1b2..4d514e48 100755 --- a/dist2/RunGrepTest +++ b/dist2/RunGrepTest @@ -8,7 +8,7 @@ # * Put printf arguments in single, not double quotes to avoid unwanted # escaping. # * Use \0 for binary zero in printf, not \x0, for the benefit of older -# versions. +# versions (and use octal for other special values). # Set the C locale, so that sort(1) behaves predictably. @@ -653,6 +653,33 @@ printf 'ABC\0XYZ\nABCDEF\nDEFABC\n' >testtemp2grep $valgrind $vjs $pcre2grep -a -f testtemp1grep testtemp2grep >>testtrygrep echo "RC=$?" >>testtrygrep +echo "---------------------------- Test 127 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -o --om-capture=0 'pattern()()()()' testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 128 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -o1 --om-capture=0 'pattern()()()()' testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 129 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -m 2 'fox' testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 130 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -o -m2 'fox' testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 131 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -oc -m2 'fox' testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 132 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -m1 -A3 '^match'; echo '---'; head -1) <$srcdir/testdata/grepinput >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 133 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $vjs $pcre2grep -m1 -O '=$x{41}$x423$o{103}$o1045=' 'fox') <$srcdir/testdata/grepinputv >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep # Now compare the results. @@ -677,6 +704,20 @@ if [ $utf8 -ne 0 ] ; then (cd $srcdir; $valgrind $vjs $pcre2grep --line-offsets -u --newline=any '(?<=\K\x{17f})' ./testdata/grepinput8) >>testtrygrep echo "RC=$?" >>testtrygrep + echo "---------------------------- Test U4 ------------------------------" >>testtrygrep + printf 'A\341\200\200\200CD\342\200\200Z\n' >testtemp1grep + (cd $srcdir; $valgrind $vjs $pcre2grep -u -o '....' $builddir/testtemp1grep) >>testtrygrep 2>&1 + echo "RC=$?" >>testtrygrep + + echo "---------------------------- Test U5 ------------------------------" >>testtrygrep + printf 'A\341\200\200\200CD\342\200\200Z\n' >testtemp1grep + (cd $srcdir; $valgrind $vjs $pcre2grep -U -o '....' $builddir/testtemp1grep) >>testtrygrep + echo "RC=$?" >>testtrygrep + + echo "---------------------------- Test U6 -----------------------------" >>testtrygrep + (cd $srcdir; $valgrind $vjs $pcre2grep -u -m1 -O '=$x{1d3}$o{744}=' 'fox') <$srcdir/testdata/grepinputv >>testtrygrep 2>&1 + echo "RC=$?" >>testtrygrep + $cf $srcdir/testdata/grepoutput8 testtrygrep if [ $? != 0 ] ; then exit 1; fi @@ -716,22 +757,20 @@ $valgrind $vjs $pcre2grep -n --newline=anycrlf "^(abc|def|ghi|jkl)" testNinputgr # It seems impossible to handle NUL characters easily in many operating # systems, including Solaris (aka SunOS), where the version of sed explicitly -# doesn't like them, and also MacOS (Darwin), OpenBSD, FreeBSD, and NetBSD. So -# now we run this test only on OS that are known to work. For the rest, we -# fudge the output so that the comparison works. +# doesn't like them, and also MacOS (Darwin), OpenBSD, FreeBSD, NetBSD, and +# some Linux distributions like Alpine, even when using GNU sed, so test for +# a usable sed and fudge the output so that the comparison works when sed +# doesn't. printf '%c--------------------------- Test N7 ------------------------------\r\n' - >>testtrygrep -uname=`uname` -case $uname in - Linux) - printf 'abc\0def' >testNinputgrep - $valgrind $vjs $pcre2grep -na --newline=nul "^(abc|def)" testNinputgrep | sed 's/\x00/ZERO/' >>testtrygrep - echo "" >>testtrygrep - ;; - *) - echo '1:abcZERO2:def' >>testtrygrep - ;; -esac +Z=`printf '\0' | sed 's/\x00/Z/g'` +if [ "$Z" = "Z" ]; then + printf 'abc\0def' >testNinputgrep + $valgrind $vjs $pcre2grep -na --newline=nul "^(abc|def)" testNinputgrep | sed 's/\x00/ZERO/g' >>testtrygrep + echo "" >>testtrygrep +else + echo '1:abcZERO2:defZERO' >>testtrygrep +fi $cf $srcdir/testdata/grepoutputN testtrygrep if [ $? != 0 ] ; then exit 1; fi @@ -747,6 +786,7 @@ if $valgrind $vjs $pcre2grep --help | $valgrind $vjs $pcre2grep -q 'callout scri $valgrind $vjs $pcre2grep '(T)(..(.))()()()()()()()(..)(?C"/bin/echo|Arg1: [$11] [${11}]")' $srcdir/testdata/grepinputv >>testtrygrep $valgrind $vjs $pcre2grep '(T)(?C"|$0:$1$n")' $srcdir/testdata/grepinputv >>testtrygrep $valgrind $vjs $pcre2grep '(T)(?C"|$1$n")(*F)' $srcdir/testdata/grepinputv >>testtrygrep + $valgrind $vjs $pcre2grep -m1 '(T)(?C"|$0:$1:$x{41}$o{101}$n")' $srcdir/testdata/grepinputv >>testtrygrep if $valgrind $vjs $pcre2grep --help | $valgrind $vjs $pcre2grep -q 'Non-fork callout scripts in patterns are supported'; then $cf $srcdir/testdata/grepoutputCN testtrygrep diff --git a/dist2/RunTest b/dist2/RunTest index 6715d153..14d9f60f 100755 --- a/dist2/RunTest +++ b/dist2/RunTest @@ -493,15 +493,20 @@ for bmode in "$test8" "$test16" "$test32"; do done fi - # PCRE2 tests that are not Perl-compatible: API, errors, internals + # PCRE2 tests that are not Perl-compatible: API, errors, internals. We copy + # the testbtables file to the current directory for use by this test. if [ $do2 = yes ] ; then echo $title2 "(excluding UTF-$bits)" + cp $testdata/testbtables . for opt in "" $jitopt; do $sim $valgrind ${opt:+$vjs} ./pcre2test -q $setstack $bmode $opt $testdata/testinput2 testtry - if [ $? = 0 ] ; then + saverc=$? + if [ $saverc = 0 ] ; then $sim $valgrind ${opt:+$vjs} ./pcre2test -q $bmode $opt -error -70,-62,-2,-1,0,100,101,191,200 >>testtry checkresult $? 2 "$opt" + else + checkresult $saverc 2 "$opt" fi done fi @@ -859,6 +864,6 @@ for bmode in "$test8" "$test16" "$test32"; do done # Clean up local working files -rm -f testSinput test3input testsaved1 testsaved2 test3output test3outputA test3outputB teststdout teststderr testtry +rm -f testbtables testSinput test3input testsaved1 testsaved2 test3output test3outputA test3outputB teststdout teststderr testtry # End diff --git a/dist2/RunTest.bat b/dist2/RunTest.bat index 9474434f..791f2650 100644 --- a/dist2/RunTest.bat +++ b/dist2/RunTest.bat @@ -26,6 +26,7 @@ @rem Updated for new test 14 (moving others up a number), August 2015.
@rem Tidied and updated for new tests 21, 22, 23 by PH, October 2015.
@rem PH added missing "set type" for test 22, April 2016.
+@rem PH added copy command for new testbtables file, November 2020
setlocal enabledelayedexpansion
@@ -305,6 +306,7 @@ if %jit% EQU 1 call :runsub 1 testoutjit "Test with JIT Override" -q -jit goto :eof
:do2
+ copy /y %srcdir%\testdata\testbtables testbtables
call :runsub 2 testout "API, errors, internals, and non-Perl stuff" -q
if %jit% EQU 1 call :runsub 2 testoutjit "Test with JIT Override" -q -jit
goto :eof
diff --git a/dist2/aclocal.m4 b/dist2/aclocal.m4 index 87e34b08..a1b8aed6 100644 --- a/dist2/aclocal.m4 +++ b/dist2/aclocal.m4 @@ -1,6 +1,6 @@ -# generated automatically by aclocal 1.16.1 -*- Autoconf -*- +# generated automatically by aclocal 1.16.2 -*- Autoconf -*- -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2020 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -364,7 +364,7 @@ AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"], [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])]) ])dnl PKG_HAVE_DEFINE_WITH_MODULES -# Copyright (C) 2002-2018 Free Software Foundation, Inc. +# Copyright (C) 2002-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -379,7 +379,7 @@ AC_DEFUN([AM_AUTOMAKE_VERSION], [am__api_version='1.16' dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to dnl require some minimum version. Point them to the right macro. -m4_if([$1], [1.16.1], [], +m4_if([$1], [1.16.2], [], [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl ]) @@ -395,12 +395,12 @@ m4_define([_AM_AUTOCONF_VERSION], []) # Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced. # This function is AC_REQUIREd by AM_INIT_AUTOMAKE. AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], -[AM_AUTOMAKE_VERSION([1.16.1])dnl +[AM_AUTOMAKE_VERSION([1.16.2])dnl m4_ifndef([AC_AUTOCONF_VERSION], [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl _AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))]) -# Copyright (C) 2011-2018 Free Software Foundation, Inc. +# Copyright (C) 2011-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -462,7 +462,7 @@ AC_SUBST([AR])dnl # AM_AUX_DIR_EXPAND -*- Autoconf -*- -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -514,7 +514,7 @@ am_aux_dir=`cd "$ac_aux_dir" && pwd` # AM_CONDITIONAL -*- Autoconf -*- -# Copyright (C) 1997-2018 Free Software Foundation, Inc. +# Copyright (C) 1997-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -545,7 +545,7 @@ AC_CONFIG_COMMANDS_PRE( Usually this means the macro was only invoked conditionally.]]) fi])]) -# Copyright (C) 1999-2018 Free Software Foundation, Inc. +# Copyright (C) 1999-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -736,7 +736,7 @@ _AM_SUBST_NOTMAKE([am__nodep])dnl # Generate code to set up dependency tracking. -*- Autoconf -*- -# Copyright (C) 1999-2018 Free Software Foundation, Inc. +# Copyright (C) 1999-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -775,7 +775,9 @@ AC_DEFUN([_AM_OUTPUT_DEPENDENCY_COMMANDS], done if test $am_rc -ne 0; then AC_MSG_FAILURE([Something went wrong bootstrapping makefile fragments - for automatic dependency tracking. Try re-running configure with the + for automatic dependency tracking. If GNU make was not used, consider + re-running the configure script with MAKE="gmake" (or whatever is + necessary). You can also try re-running configure with the '--disable-dependency-tracking' option to at least be able to build the package (albeit without support for automatic dependency tracking).]) fi @@ -802,7 +804,7 @@ AC_DEFUN([AM_OUTPUT_DEPENDENCY_COMMANDS], # Do all the work for Automake. -*- Autoconf -*- -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -999,7 +1001,7 @@ for _am_header in $config_headers :; do done echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count]) -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1020,7 +1022,7 @@ if test x"${install_sh+set}" != xset; then fi AC_SUBST([install_sh])]) -# Copyright (C) 2003-2018 Free Software Foundation, Inc. +# Copyright (C) 2003-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1041,7 +1043,7 @@ AC_SUBST([am__leading_dot])]) # Check to see how 'make' treats includes. -*- Autoconf -*- -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1084,7 +1086,7 @@ AC_SUBST([am__quote])]) # Fake the existence of programs that GNU maintainers use. -*- Autoconf -*- -# Copyright (C) 1997-2018 Free Software Foundation, Inc. +# Copyright (C) 1997-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1123,7 +1125,7 @@ fi # Helper functions for option handling. -*- Autoconf -*- -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1152,7 +1154,7 @@ AC_DEFUN([_AM_SET_OPTIONS], AC_DEFUN([_AM_IF_OPTION], [m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])]) -# Copyright (C) 1999-2018 Free Software Foundation, Inc. +# Copyright (C) 1999-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1199,7 +1201,7 @@ AC_LANG_POP([C])]) # For backward compatibility. AC_DEFUN_ONCE([AM_PROG_CC_C_O], [AC_REQUIRE([AC_PROG_CC])]) -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1218,7 +1220,7 @@ AC_DEFUN([AM_RUN_LOG], # Check to make sure that the build environment is sane. -*- Autoconf -*- -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1299,7 +1301,7 @@ AC_CONFIG_COMMANDS_PRE( rm -f conftest.file ]) -# Copyright (C) 2009-2018 Free Software Foundation, Inc. +# Copyright (C) 2009-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1359,7 +1361,7 @@ AC_SUBST([AM_BACKSLASH])dnl _AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl ]) -# Copyright (C) 2001-2018 Free Software Foundation, Inc. +# Copyright (C) 2001-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1387,7 +1389,7 @@ fi INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s" AC_SUBST([INSTALL_STRIP_PROGRAM])]) -# Copyright (C) 2006-2018 Free Software Foundation, Inc. +# Copyright (C) 2006-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -1406,7 +1408,7 @@ AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)]) # Check how to create a tarball. -*- Autoconf -*- -# Copyright (C) 2004-2018 Free Software Foundation, Inc. +# Copyright (C) 2004-2020 Free Software Foundation, Inc. # # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, diff --git a/dist2/ar-lib b/dist2/ar-lib index 0baa4f60..1e9388e2 100755 --- a/dist2/ar-lib +++ b/dist2/ar-lib @@ -2,9 +2,9 @@ # Wrapper for Microsoft lib.exe me=ar-lib -scriptversion=2012-03-01.08; # UTC +scriptversion=2019-07-04.01; # UTC -# Copyright (C) 2010-2018 Free Software Foundation, Inc. +# Copyright (C) 2010-2020 Free Software Foundation, Inc. # Written by Peter Rosin <peda@lysator.liu.se>. # # This program is free software; you can redistribute it and/or modify @@ -53,7 +53,7 @@ func_file_conv () MINGW*) file_conv=mingw ;; - CYGWIN*) + CYGWIN* | MSYS*) file_conv=cygwin ;; *) @@ -65,7 +65,7 @@ func_file_conv () mingw) file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` ;; - cygwin) + cygwin | msys) file=`cygpath -m "$file" || echo "$file"` ;; wine) @@ -224,10 +224,11 @@ elif test -n "$extract"; then esac done else - $AR -NOLOGO -LIST "$archive" | sed -e 's/\\/\\\\/g' | while read member - do - $AR -NOLOGO -EXTRACT:"$member" "$archive" || exit $? - done + $AR -NOLOGO -LIST "$archive" | tr -d '\r' | sed -e 's/\\/\\\\/g' \ + | while read member + do + $AR -NOLOGO -EXTRACT:"$member" "$archive" || exit $? + done fi elif test -n "$quick$replace"; then diff --git a/dist2/compile b/dist2/compile index 99e50524..23fcba01 100755 --- a/dist2/compile +++ b/dist2/compile @@ -3,7 +3,7 @@ scriptversion=2018-03-07.03; # UTC -# Copyright (C) 1999-2018 Free Software Foundation, Inc. +# Copyright (C) 1999-2020 Free Software Foundation, Inc. # Written by Tom Tromey <tromey@cygnus.com>. # # This program is free software; you can redistribute it and/or modify @@ -53,7 +53,7 @@ func_file_conv () MINGW*) file_conv=mingw ;; - CYGWIN*) + CYGWIN* | MSYS*) file_conv=cygwin ;; *) @@ -67,7 +67,7 @@ func_file_conv () mingw/*) file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` ;; - cygwin/*) + cygwin/* | msys/*) file=`cygpath -m "$file" || echo "$file"` ;; wine/*) diff --git a/dist2/config-cmake.h.in b/dist2/config-cmake.h.in index 44d51ae8..7766dd74 100644 --- a/dist2/config-cmake.h.in +++ b/dist2/config-cmake.h.in @@ -11,7 +11,10 @@ #cmakedefine HAVE_WINDOWS_H 1 #cmakedefine HAVE_BCOPY 1 +#cmakedefine HAVE_MEMFD_CREATE 1 #cmakedefine HAVE_MEMMOVE 1 +#cmakedefine HAVE_SECURE_GETENV 1 +#cmakedefine HAVE_STRERROR 1 #cmakedefine PCRE2_STATIC 1 diff --git a/dist2/config.guess b/dist2/config.guess index 256083a7..45001cfe 100755 --- a/dist2/config.guess +++ b/dist2/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-03-08' +timestamp='2020-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -84,8 +84,6 @@ if test $# != 0; then exit 1 fi -trap 'exit 1' 1 2 15 - # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a @@ -96,34 +94,40 @@ trap 'exit 1' 1 2 15 # Portable tmp directory creation inspired by the Autoconf team. -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > "$dummy.c" ; - for c in cc gcc c89 c99 ; do - if ($c -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ; set_cc_for_build= ;' +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 + +set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then +if test -f /.attbin/uname ; then PATH=$PATH:/.attbin ; export PATH fi @@ -138,7 +142,7 @@ Linux|GNU|GNU/*) # We could probably try harder. LIBC=gnu - eval "$set_cc_for_build" + set_cc_for_build cat <<-EOF > "$dummy.c" #include <features.h> #if defined(__UCLIBC__) @@ -199,7 +203,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in os=netbsdelf ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval "$set_cc_for_build" + set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -237,7 +241,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "$machine-${os}${release}${abi}" + echo "$machine-${os}${release}${abi-}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -260,6 +264,9 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:SolidBSD:*:*) echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; + *:OS108:*:*) + echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" + exit ;; macppc:MirBSD:*:*) echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; @@ -269,12 +276,15 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:Sortix:*:*) echo "$UNAME_MACHINE"-unknown-sortix exit ;; + *:Twizzler:*:*) + echo "$UNAME_MACHINE"-unknown-twizzler + exit ;; *:Redox:*:*) echo "$UNAME_MACHINE"-unknown-redox exit ;; mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -389,7 +399,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval "$set_cc_for_build" + set_cc_for_build SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. @@ -482,7 +492,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval "$set_cc_for_build" + set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include <stdio.h> /* for printf() prototype */ @@ -579,7 +589,7 @@ EOF exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval "$set_cc_for_build" + set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" #include <sys/systemcfg.h> @@ -660,7 +670,7 @@ EOF esac fi if [ "$HP_ARCH" = "" ]; then - eval "$set_cc_for_build" + set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE @@ -700,7 +710,7 @@ EOF esac if [ "$HP_ARCH" = hppa2.0w ] then - eval "$set_cc_for_build" + set_cc_for_build # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -726,7 +736,7 @@ EOF echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval "$set_cc_for_build" + set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" #include <unistd.h> int @@ -840,6 +850,17 @@ EOF *:BSD/OS:*:*) echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" exit ;; + arm:FreeBSD:*:*) + UNAME_PROCESSOR=`uname -p` + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + fi + exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` case "$UNAME_PROCESSOR" in @@ -881,7 +902,7 @@ EOF echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-unknown-cygwin + echo x86_64-pc-cygwin exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" @@ -894,8 +915,8 @@ EOF # other systems with GNU libc and userland echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; - i*86:Minix:*:*) - echo "$UNAME_MACHINE"-pc-minix + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix exit ;; aarch64:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -905,7 +926,7 @@ EOF echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null` in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -922,7 +943,7 @@ EOF echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval "$set_cc_for_build" + set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then @@ -971,23 +992,51 @@ EOF echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval "$set_cc_for_build" + set_cc_for_build + IS_GLIBC=0 + test x"${LIBC}" = xgnu && IS_GLIBC=1 sed 's/^ //' << EOF > "$dummy.c" #undef CPU - #undef ${UNAME_MACHINE} - #undef ${UNAME_MACHINE}el + #undef mips + #undef mipsel + #undef mips64 + #undef mips64el + #if ${IS_GLIBC} && defined(_ABI64) + LIBCABI=gnuabi64 + #else + #if ${IS_GLIBC} && defined(_ABIN32) + LIBCABI=gnuabin32 + #else + LIBCABI=${LIBC} + #endif + #endif + + #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa64r6 + #else + #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 + CPU=mipsisa32r6 + #else + #if defined(__mips64) + CPU=mips64 + #else + CPU=mips + #endif + #endif + #endif + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=${UNAME_MACHINE}el + MIPS_ENDIAN=el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=${UNAME_MACHINE} + MIPS_ENDIAN= #else - CPU= + MIPS_ENDIAN= #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" - test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI'`" + test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; mips64el:Linux:*:*) echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" @@ -1100,7 +1149,7 @@ EOF *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then @@ -1284,38 +1333,39 @@ EOF echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - eval "$set_cc_for_build" - if test "$UNAME_PROCESSOR" = unknown ; then - UNAME_PROCESSOR=powerpc + UNAME_PROCESSOR=`uname -p` + case $UNAME_PROCESSOR in + unknown) UNAME_PROCESSOR=powerpc ;; + esac + if command -v xcode-select > /dev/null 2> /dev/null && \ + ! xcode-select --print-path > /dev/null 2> /dev/null ; then + # Avoid executing cc if there is no toolchain installed as + # cc will be a stub that puts up a graphical alert + # prompting the user to install developer tools. + CC_FOR_BUILD=no_compiler_found + else + set_cc_for_build fi - if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then - if [ "$CC_FOR_BUILD" != no_compiler_found ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc fi elif test "$UNAME_PROCESSOR" = i386 ; then - # Avoid executing cc on OS X 10.9, as it ships with a stub - # that puts up a graphical alert prompting to install - # developer tools. Any system running Mac OS X 10.7 or - # later (Darwin 11 and later) is required to have a 64-bit - # processor. This is not true of the ARM version of Darwin - # that Apple uses in portable devices. - UNAME_PROCESSOR=x86_64 + # uname -m returns i386 or x86_64 + UNAME_PROCESSOR=$UNAME_MACHINE fi echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; @@ -1358,6 +1408,7 @@ EOF # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. + # shellcheck disable=SC2154 if test "$cputype" = 386; then UNAME_MACHINE=i386 else @@ -1414,8 +1465,148 @@ EOF amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs exit ;; + *:Unleashed:*:*) + echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" + exit ;; esac +# No uname command or uname output not recognized. +set_cc_for_build +cat > "$dummy.c" <<EOF +#ifdef _SEQUENT_ +#include <sys/types.h> +#include <sys/utsname.h> +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#include <signal.h> +#if defined(_SIZE_T_) || defined(SIGLOST) +#include <sys/utsname.h> +#endif +#endif +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include <sys/param.h> + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); +#endif + +#if defined (vax) +#if !defined (ultrix) +#include <sys/param.h> +#if defined (BSD) +#if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +#else +#if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#endif +#else + printf ("vax-dec-bsd\n"); exit (0); +#endif +#else +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname un; + uname (&un); + printf ("vax-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("vax-dec-ultrix\n"); exit (0); +#endif +#endif +#endif +#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) +#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) +#if defined(_SIZE_T_) || defined(SIGLOST) + struct utsname *un; + uname (&un); + printf ("mips-dec-ultrix%s\n", un.release); exit (0); +#else + printf ("mips-dec-ultrix\n"); exit (0); +#endif +#endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } + +# Apollos put the system type in the environment. +test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } + echo "$0: unable to guess system type" >&2 case "$UNAME_MACHINE:$UNAME_SYSTEM" in diff --git a/dist2/config.sub b/dist2/config.sub index 9ccf09a7..f02d43ad 100755 --- a/dist2/config.sub +++ b/dist2/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2018 Free Software Foundation, Inc. +# Copyright 1992-2020 Free Software Foundation, Inc. -timestamp='2018-03-08' +timestamp='2020-01-01' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2018 Free Software Foundation, Inc. +Copyright 1992-2020 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -89,7 +89,7 @@ while test $# -gt 0 ; do - ) # Use stdin as input. break ;; -* ) - echo "$me: invalid option $1$help" + echo "$me: invalid option $1$help" >&2 exit 1 ;; *local*) @@ -110,1223 +110,1164 @@ case $# in exit 1;; esac -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ - kopensolaris*-gnu* | cloudabi*-eabi* | \ - storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - android-linux) - os=-linux-android - basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown - ;; - *) - basic_machine=`echo "$1" | sed 's/-[^-]*$//'` - if [ "$basic_machine" != "$1" ] - then os=`echo "$1" | sed 's/.*-/-/'` - else os=; fi - ;; -esac +# Split fields of configuration type +# shellcheck disable=SC2162 +IFS="-" read field1 field2 field3 field4 <<EOF +$1 +EOF -### Let's recognize common machines as not being operating systems so -### that things like config.sub decstation-3100 work. We also -### recognize some manufacturers as not being operating systems, so we -### can provide default operating systems below. -case $os in - -sun*os*) - # Prevent following clause from handling this invalid input. - ;; - -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ - -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ - -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ - -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ - -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ - -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis | -knuth | -cray | -microblaze*) - os= - basic_machine=$1 - ;; - -bluegene*) - os=-cnk - ;; - -sim | -cisco | -oki | -wec | -winbond) - os= - basic_machine=$1 - ;; - -scout) - ;; - -wrs) - os=-vxworks - basic_machine=$1 - ;; - -chorusos*) - os=-chorusos - basic_machine=$1 - ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 - ;; - -hiux*) - os=-hiuxwe2 - ;; - -sco6) - os=-sco5v6 - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco5v6*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -udk*) - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -isc) - os=-isc2.2 - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -clix*) - basic_machine=clipper-intergraph - ;; - -isc*) - basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` - ;; - -lynx*178) - os=-lynxos178 - ;; - -lynx*5) - os=-lynxos5 +# Separate into logical components for further validation +case $1 in + *-*-*-*-*) + echo Invalid configuration \`"$1"\': more than four components >&2 + exit 1 ;; - -lynx*) - os=-lynxos + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 ;; - -ptx*) - basic_machine=`echo "$1" | sed -e 's/86-.*/86-sequent/'` + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac ;; - -psos*) - os=-psos + *-*) + # A lone config we happen to match not fitting any pattern + case $field1-$field2 in + decstation-3100) + basic_machine=mips-dec + os= + ;; + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* \ + | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ + | ultra | tti* | harris | dolphin | highlevel | gould \ + | cbm | ns | masscomp | apple | axis | knuth | cray \ + | microblaze* | sim | cisco \ + | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac + ;; + esac ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amiga) + basic_machine=m68k-unknown + os= + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + da30) + basic_machine=m68k-da30 + os= + ;; + decstation | pmax | pmin | dec3100 | decstatn) + basic_machine=mips-dec + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300 | hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sequent) + basic_machine=i386-sequent + os= + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac ;; esac -# Decode aliases for certain CPU-COMPANY combinations. +# Decode 1-component or ad-hoc basic machines case $basic_machine in - # Recognize the basic CPU types without company name. - # Some are omitted here because they have special meanings below. - 1750a | 580 \ - | a29k \ - | aarch64 | aarch64_be \ - | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ - | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ - | am33_2.0 \ - | arc | arceb \ - | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ - | avr | avr32 \ - | ba \ - | be32 | be64 \ - | bfin \ - | c4x | c8051 | clipper \ - | d10v | d30v | dlx | dsp16xx \ - | e2k | epiphany \ - | fido | fr30 | frv | ft32 \ - | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ - | hexagon \ - | i370 | i860 | i960 | ia16 | ia64 \ - | ip2k | iq2000 \ - | k1om \ - | le32 | le64 \ - | lm32 \ - | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64el \ - | mips64octeon | mips64octeonel \ - | mips64orion | mips64orionel \ - | mips64r5900 | mips64r5900el \ - | mips64vr | mips64vrel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mips64vr5900 | mips64vr5900el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa32r6 | mipsisa32r6el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64r6 | mipsisa64r6el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipsr5900 | mipsr5900el \ - | mipstx39 | mipstx39el \ - | mn10200 | mn10300 \ - | moxie \ - | mt \ - | msp430 \ - | nds32 | nds32le | nds32be \ - | nios | nios2 | nios2eb | nios2el \ - | ns16k | ns32k \ - | open8 | or1k | or1knd | or32 \ - | pdp10 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle \ - | pru \ - | pyramid \ - | riscv32 | riscv64 \ - | rl78 | rx \ - | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ - | sh64 | sh64le \ - | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ - | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ - | spu \ - | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ - | ubicom32 \ - | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ - | visium \ - | wasm32 \ - | x86 | xc16x | xstormy16 | xtensa \ - | z8k | z80) - basic_machine=$basic_machine-unknown - ;; - c54x) - basic_machine=tic54x-unknown - ;; - c55x) - basic_machine=tic55x-unknown - ;; - c6x) - basic_machine=tic6x-unknown - ;; - leon|leon[3-9]) - basic_machine=sparc-$basic_machine - ;; - m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip) - basic_machine=$basic_machine-unknown - os=-none + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + w89k) + cpu=hppa1.1 + vendor=winbond ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) + op50n) + cpu=hppa1.1 + vendor=oki ;; - ms1) - basic_machine=mt-unknown + op60c) + cpu=hppa1.1 + vendor=oki ;; - - strongarm | thumb | xscale) - basic_machine=arm-unknown + ibm*) + cpu=i370 + vendor=ibm ;; - xgate) - basic_machine=$basic_machine-unknown - os=-none + orion105) + cpu=clipper + vendor=highlevel ;; - xscaleeb) - basic_machine=armeb-unknown + mac | mpw | mac-mpw) + cpu=m68k + vendor=apple ;; - - xscaleel) - basic_machine=armel-unknown + pmac | pmac-mpw) + cpu=powerpc + vendor=apple ;; - # We use `pc' rather than `unknown' - # because (1) that's what they normally are, and - # (2) the word "unknown" tends to confuse beginning users. - i*86 | x86_64) - basic_machine=$basic_machine-pc - ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 - exit 1 - ;; - # Recognize the basic CPU types with company name. - 580-* \ - | a29k-* \ - | aarch64-* | aarch64_be-* \ - | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ - | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* | avr32-* \ - | ba-* \ - | be32-* | be64-* \ - | bfin-* | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | c8051-* | clipper-* | craynv-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | e2k-* | elxsi-* \ - | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ - | h8300-* | h8500-* \ - | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ - | hexagon-* \ - | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ - | ip2k-* | iq2000-* \ - | k1om-* \ - | le32-* | le64-* \ - | lm32-* \ - | m32c-* | m32r-* | m32rle-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ - | microblaze-* | microblazeel-* \ - | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ - | mips16-* \ - | mips64-* | mips64el-* \ - | mips64octeon-* | mips64octeonel-* \ - | mips64orion-* | mips64orionel-* \ - | mips64r5900-* | mips64r5900el-* \ - | mips64vr-* | mips64vrel-* \ - | mips64vr4100-* | mips64vr4100el-* \ - | mips64vr4300-* | mips64vr4300el-* \ - | mips64vr5000-* | mips64vr5000el-* \ - | mips64vr5900-* | mips64vr5900el-* \ - | mipsisa32-* | mipsisa32el-* \ - | mipsisa32r2-* | mipsisa32r2el-* \ - | mipsisa32r6-* | mipsisa32r6el-* \ - | mipsisa64-* | mipsisa64el-* \ - | mipsisa64r2-* | mipsisa64r2el-* \ - | mipsisa64r6-* | mipsisa64r6el-* \ - | mipsisa64sb1-* | mipsisa64sb1el-* \ - | mipsisa64sr71k-* | mipsisa64sr71kel-* \ - | mipsr5900-* | mipsr5900el-* \ - | mipstx39-* | mipstx39el-* \ - | mmix-* \ - | mt-* \ - | msp430-* \ - | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* | nios2eb-* | nios2el-* \ - | none-* | np1-* | ns16k-* | ns32k-* \ - | open8-* \ - | or1k*-* \ - | orion-* \ - | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ - | pru-* \ - | pyramid-* \ - | riscv32-* | riscv64-* \ - | rl78-* | romp-* | rs6000-* | rx-* \ - | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ - | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ - | tahoe-* \ - | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tile*-* \ - | tron-* \ - | ubicom32-* \ - | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ - | vax-* \ - | visium-* \ - | wasm32-* \ - | we32k-* \ - | x86-* | x86_64-* | xc16x-* | xps100-* \ - | xstormy16-* | xtensa*-* \ - | ymp-* \ - | z8k-* | z80-*) - ;; - # Recognize the basic CPU types without company name, with glob match. - xtensa*) - basic_machine=$basic_machine-unknown - ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-pc - os=-bsd - ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - basic_machine=m68000-att + cpu=m68000 + vendor=att ;; 3b*) - basic_machine=we32k-att - ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; - amd64) - basic_machine=x86_64-pc - ;; - amd64-*) - basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv - ;; - amiga | amiga-*) - basic_machine=m68k-unknown - ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aros) - basic_machine=i386-pc - os=-aros - ;; - asmjs) - basic_machine=asmjs-unknown - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - blackfin) - basic_machine=bfin-unknown - os=-linux - ;; - blackfin-*) - basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` - os=-linux + cpu=we32k + vendor=att ;; bluegene*) - basic_machine=powerpc-ibm - os=-cnk - ;; - c54x-*) - basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - c55x-*) - basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - c6x-*) - basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - c90) - basic_machine=c90-cray - os=-unicos - ;; - cegcc) - basic_machine=arm-unknown - os=-cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - craynv) - basic_machine=craynv-cray - os=-unicosmp - ;; - cr16 | cr16-*) - basic_machine=cr16-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds - ;; - crisv32 | crisv32-* | etraxfs*) - basic_machine=crisv32-axis - ;; - cris | cris-* | etrax*) - basic_machine=cris-axis - ;; - crx) - basic_machine=crx-unknown - os=-elf - ;; - da30 | da30-*) - basic_machine=m68k-da30 - ;; - decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) - basic_machine=mips-dec + cpu=powerpc + vendor=ibm + os=cnk ;; decsystem10* | dec10*) - basic_machine=pdp10-dec - os=-tops10 + cpu=pdp10 + vendor=dec + os=tops10 ;; decsystem20* | dec20*) - basic_machine=pdp10-dec - os=-tops20 + cpu=pdp10 + vendor=dec + os=tops20 ;; delta | 3300 | motorola-3300 | motorola-delta \ | 3300-motorola | delta-motorola) - basic_machine=m68k-motorola - ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dicos) - basic_machine=i686-pc - os=-dicos - ;; - djgpp) - basic_machine=i586-pc - os=-msdosdjgpp - ;; - dpx20 | dpx20-*) - basic_machine=rs6000-bull - os=-bosx + cpu=m68k + vendor=motorola ;; dpx2*) - basic_machine=m68k-bull - os=-sysv3 - ;; - e500v[12]) - basic_machine=powerpc-unknown - os=$os"spe" - ;; - e500v[12]-*) - basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` - os=$os"spe" - ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon - ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd + cpu=m68k + vendor=bull + os=sysv3 ;; encore | umax | mmax) - basic_machine=ns32k-encore + cpu=ns32k + vendor=encore ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose + elxsi) + cpu=elxsi + vendor=elxsi + os=${os:-bsd} ;; fx2800) - basic_machine=i860-alliant + cpu=i860 + vendor=alliant ;; genix) - basic_machine=ns32k-ns - ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 + cpu=ns32k + vendor=ns ;; h3050r* | hiux*) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 - ;; - hp300-*) - basic_machine=m68k-hp - ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux + cpu=hppa1.1 + vendor=hitachi + os=hiuxwe2 ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k2[0-9][0-9] | hp9k31[0-9]) - basic_machine=m68000-hp + cpu=m68000 + vendor=hp ;; hp9k3[2-9][0-9]) - basic_machine=m68k-hp + cpu=m68k + vendor=hp ;; hp9k6[0-9][0-9] | hp6[0-9][0-9]) - basic_machine=hppa1.0-hp + cpu=hppa1.0 + vendor=hp ;; hp9k7[0-79][0-9] | hp7[0-79][0-9]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k78[0-9] | hp78[0-9]) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][13679] | hp8[0-9][13679]) - basic_machine=hppa1.1-hp + cpu=hppa1.1 + vendor=hp ;; hp9k8[0-9][0-9] | hp8[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm + cpu=hppa1.0 + vendor=hp ;; i*86v32) - basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` - os=-sysv32 + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv32 ;; i*86v4*) - basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` - os=-sysv4 + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv4 ;; i*86v) - basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` - os=-sysv + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=sysv ;; i*86sol2) - basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` - os=-solaris2 - ;; - i386mach) - basic_machine=i386-mach - os=-mach + cpu=`echo "$1" | sed -e 's/86.*/86/'` + vendor=pc + os=solaris2 ;; - vsta) - basic_machine=i386-unknown - os=-vsta + j90 | j90-cray) + cpu=j90 + vendor=cray + os=${os:-unicos} ;; iris | iris4d) - basic_machine=mips-sgi + cpu=mips + vendor=sgi case $os in - -irix*) + irix*) ;; *) - os=-irix4 + os=irix4 ;; esac ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - leon-*|leon[3-9]-*) - basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` - ;; - m68knommu) - basic_machine=m68k-unknown - os=-linux - ;; - m68knommu-*) - basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` - os=-linux - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv - ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - microblaze*) - basic_machine=microblaze-xilinx - ;; - mingw64) - basic_machine=x86_64-pc - os=-mingw64 - ;; - mingw32) - basic_machine=i686-pc - os=-mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - os=-mingw32ce - ;; miniframe) - basic_machine=m68000-convergent - ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) - basic_machine=m68k-atari - os=-mint - ;; - mips3*-*) - basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` - ;; - mips3*) - basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - moxiebox) - basic_machine=moxie-unknown - os=-moxiebox - ;; - msdos) - basic_machine=i386-pc - os=-msdos - ;; - ms1-*) - basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` - ;; - msys) - basic_machine=i686-pc - os=-msys - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - nacl) - basic_machine=le32-unknown - os=-nacl - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 + cpu=m68000 + vendor=convergent ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) + cpu=m68k + vendor=atari + os=mint ;; news-3600 | risc-news) - basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv + cpu=mips + vendor=sony + os=newsos ;; next | m*-next) - basic_machine=m68k-next + cpu=m68k + vendor=next case $os in - -nextstep* ) + openstep*) + ;; + nextstep*) ;; - -ns2*) - os=-nextstep2 + ns2*) + os=nextstep2 ;; *) - os=-nextstep3 + os=nextstep3 ;; esac ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; np1) - basic_machine=np1-gould - ;; - neo-tandem) - basic_machine=neo-tandem - ;; - nse-tandem) - basic_machine=nse-tandem - ;; - nsr-tandem) - basic_machine=nsr-tandem - ;; - nsv-tandem) - basic_machine=nsv-tandem - ;; - nsx-tandem) - basic_machine=nsx-tandem + cpu=np1 + vendor=gould ;; op50n-* | op60c-*) - basic_machine=hppa1.1-oki - os=-proelf - ;; - openrisc | openrisc-*) - basic_machine=or32-unknown - ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k + cpu=hppa1.1 + vendor=oki + os=proelf ;; pa-hitachi) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - parisc) - basic_machine=hppa-unknown - os=-linux - ;; - parisc-*) - basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` - os=-linux + cpu=hppa1.1 + vendor=hitachi + os=hiuxwe2 ;; pbd) - basic_machine=sparc-tti + cpu=sparc + vendor=tti ;; pbb) - basic_machine=m68k-tti + cpu=m68k + vendor=tti ;; - pc532 | pc532-*) - basic_machine=ns32k-pc532 - ;; - pc98) - basic_machine=i386-pc - ;; - pc98-*) - basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - pentium | p5 | k5 | k6 | nexgen | viac3) - basic_machine=i586-pc - ;; - pentiumpro | p6 | 6x86 | athlon | athlon_*) - basic_machine=i686-pc - ;; - pentiumii | pentium2 | pentiumiii | pentium3) - basic_machine=i686-pc - ;; - pentium4) - basic_machine=i786-pc - ;; - pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - pentium4-*) - basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` + pc532) + cpu=ns32k + vendor=pc532 ;; pn) - basic_machine=pn-gould + cpu=pn + vendor=gould ;; - power) basic_machine=power-ibm + power) + cpu=power + vendor=ibm ;; - ppc | ppcbe) basic_machine=powerpc-unknown - ;; - ppc-* | ppcbe-*) - basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - ppcle | powerpclittle) - basic_machine=powerpcle-unknown - ;; - ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` - ;; - ppc64) basic_machine=powerpc64-unknown + ps2) + cpu=i386 + vendor=ibm ;; - ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` + rm[46]00) + cpu=mips + vendor=siemens ;; - ppc64le | powerpc64little) - basic_machine=powerpc64le-unknown + rtpc | rtpc-*) + cpu=romp + vendor=ibm ;; - ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` + sde) + cpu=mipsisa32 + vendor=sde + os=${os:-elf} ;; - ps2) - basic_machine=i386-ibm + simso-wrs) + cpu=sparclite + vendor=wrs + os=vxworks ;; - pw32) - basic_machine=i586-unknown - os=-pw32 + tower | tower-32) + cpu=m68k + vendor=ncr ;; - rdos | rdos64) - basic_machine=x86_64-pc - os=-rdos + vpp*|vx|vx-*) + cpu=f301 + vendor=fujitsu ;; - rdos32) - basic_machine=i386-pc - os=-rdos + w65) + cpu=w65 + vendor=wdc ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff + w89k-*) + cpu=hppa1.1 + vendor=winbond + os=proelf ;; - rm[46]00) - basic_machine=mips-siemens + none) + cpu=none + vendor=none ;; - rtpc | rtpc-*) - basic_machine=romp-ibm + leon|leon[3-9]) + cpu=sparc + vendor=$basic_machine ;; - s390 | s390-*) - basic_machine=s390-ibm + leon-*|leon[3-9]-*) + cpu=sparc + vendor=`echo "$basic_machine" | sed 's/-.*//'` ;; - s390x | s390x-*) - basic_machine=s390x-ibm + + *-*) + # shellcheck disable=SC2162 + IFS="-" read cpu vendor <<EOF +$basic_machine +EOF ;; - sa29200) - basic_machine=a29k-amd - os=-udi + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + cpu=$basic_machine + vendor=pc ;; - sb1) - basic_machine=mipsisa64sb1-unknown + # These rules are duplicated from below for sake of the special case above; + # i.e. things that normalized to x86 arches should also default to "pc" + pc98) + cpu=i386 + vendor=pc ;; - sb1el) - basic_machine=mipsisa64sb1el-unknown + x64 | amd64) + cpu=x86_64 + vendor=pc ;; - sde) - basic_machine=mipsisa32-sde - os=-elf + # Recognize the basic CPU types without company name. + *) + cpu=$basic_machine + vendor=unknown ;; - sei) - basic_machine=mips-sei - os=-seiux +esac + +unset -v basic_machine + +# Decode basic machines in the full and proper CPU-Company form. +case $cpu-$vendor in + # Here we handle the default manufacturer of certain CPU types in canonical form. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv-unknown) + vendor=cray + os=${os:-unicosmp} ;; - sequent) - basic_machine=i386-sequent + c90-unknown | c90-cray) + vendor=cray + os=${os:-unicos} ;; - sh5el) - basic_machine=sh5le-unknown + fx80-unknown) + vendor=alliant ;; - simso-wrs) - basic_machine=sparclite-wrs - os=-vxworks + romp-unknown) + vendor=ibm ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 + mmix-unknown) + vendor=knuth ;; - spur) - basic_machine=spur-unknown + microblaze-unknown | microblazeel-unknown) + vendor=xilinx ;; - st2000) - basic_machine=m68k-tandem + rs6000-unknown) + vendor=ibm ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 + vax-unknown) + vendor=dec ;; - strongarm-* | thumb-*) - basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` + pdp11-unknown) + vendor=dec ;; - sun2) - basic_machine=m68000-sun + we32k-unknown) + vendor=att ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 + cydra-unknown) + vendor=cydrome ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 + i370-ibm*) + vendor=ibm ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 + orion-unknown) + vendor=highlevel ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 + xps-unknown | xps100-unknown) + cpu=xps100 + vendor=honeywell ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 + + # Here we normalize CPU types with a missing or matching vendor + dpx20-unknown | dpx20-bull) + cpu=rs6000 + vendor=bull + os=${os:-bosx} ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 + + # Here we normalize CPU types irrespective of the vendor + amd64-*) + cpu=x86_64 ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 + blackfin-*) + cpu=bfin + os=linux ;; - sun3 | sun3-*) - basic_machine=m68k-sun + c54x-*) + cpu=tic54x ;; - sun4) - basic_machine=sparc-sun + c55x-*) + cpu=tic55x ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun + c6x-*) + cpu=tic6x ;; - sv1) - basic_machine=sv1-cray - os=-unicos + e500v[12]-*) + cpu=powerpc + os=$os"spe" ;; - symmetry) - basic_machine=i386-sequent - os=-dynix + mips3*-*) + cpu=mips64 ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos + ms1-*) + cpu=mt ;; - t90) - basic_machine=t90-cray - os=-unicos + m68knommu-*) + cpu=m68k + os=linux ;; - tile*) - basic_machine=$basic_machine-unknown - os=-linux-gnu + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + cpu=s12z ;; - tx39) - basic_machine=mipstx39-unknown + openrisc-*) + cpu=or32 ;; - tx39el) - basic_machine=mipstx39el-unknown + parisc-*) + cpu=hppa + os=linux ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + cpu=i586 ;; - tower | tower-32) - basic_machine=m68k-ncr + pentiumpro-* | p6-* | 6x86-* | athlon-* | athalon_*-*) + cpu=i686 ;; - tpf) - basic_machine=s390x-ibm - os=-tpf + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + cpu=i686 ;; - udi29k) - basic_machine=a29k-amd - os=-udi + pentium4-*) + cpu=i786 ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 + pc98-*) + cpu=i386 ;; - v810 | necv810) - basic_machine=v810-nec - os=-none + ppc-* | ppcbe-*) + cpu=powerpc ;; - vaxv) - basic_machine=vax-dec - os=-sysv + ppcle-* | powerpclittle-*) + cpu=powerpcle ;; - vms) - basic_machine=vax-dec - os=-vms + ppc64-*) + cpu=powerpc64 ;; - vpp*|vx|vx-*) - basic_machine=f301-fujitsu + ppc64le-* | powerpc64little-*) + cpu=powerpc64le ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks + sb1-*) + cpu=mipsisa64sb1 ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks + sb1el-*) + cpu=mipsisa64sb1el ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks + sh5e[lb]-*) + cpu=`echo "$cpu" | sed 's/^\(sh.\)e\(.\)$/\1\2e/'` ;; - w65*) - basic_machine=w65-wdc - os=-none + spur-*) + cpu=spur ;; - w89k-*) - basic_machine=hppa1.1-winbond - os=-proelf + strongarm-* | thumb-*) + cpu=arm ;; - x64) - basic_machine=x86_64-pc + tx39-*) + cpu=mipstx39 ;; - xbox) - basic_machine=i686-pc - os=-mingw32 + tx39el-*) + cpu=mipstx39el ;; - xps | xps100) - basic_machine=xps100-honeywell + x64-*) + cpu=x86_64 ;; xscale-* | xscalee[bl]-*) - basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` - ;; - ymp) - basic_machine=ymp-cray - os=-unicos - ;; - none) - basic_machine=none-none - os=-none + cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond - ;; - op50n) - basic_machine=hppa1.1-oki - ;; - op60c) - basic_machine=hppa1.1-oki - ;; - romp) - basic_machine=romp-ibm + # Recognize the canonical CPU Types that limit and/or modify the + # company names they are paired with. + cr16-*) + os=${os:-elf} ;; - mmix) - basic_machine=mmix-knuth + crisv32-* | etraxfs*-*) + cpu=crisv32 + vendor=axis ;; - rs6000) - basic_machine=rs6000-ibm + cris-* | etrax*-*) + cpu=cris + vendor=axis ;; - vax) - basic_machine=vax-dec + crx-*) + os=${os:-elf} ;; - pdp11) - basic_machine=pdp11-dec - ;; - we32k) - basic_machine=we32k-att + neo-tandem) + cpu=neo + vendor=tandem ;; - sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown + nse-tandem) + cpu=nse + vendor=tandem ;; - cydra) - basic_machine=cydra-cydrome + nsr-tandem) + cpu=nsr + vendor=tandem ;; - orion) - basic_machine=orion-highlevel + nsv-tandem) + cpu=nsv + vendor=tandem ;; - orion105) - basic_machine=clipper-highlevel + nsx-tandem) + cpu=nsx + vendor=tandem ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple + s390-*) + cpu=s390 + vendor=ibm ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple + s390x-*) + cpu=s390x + vendor=ibm ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. + tile*-*) + os=${os:-linux-gnu} ;; + *) - echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 - exit 1 + # Recognize the canonical CPU types that are allowed with any + # company name. + case $cpu in + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ + | alphapca5[67] | alpha64pca5[67] \ + | am33_2.0 \ + | amdgcn \ + | arc | arceb \ + | arm | arm[lb]e | arme[lb] | armv* \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin | bpf | bs2000 \ + | c[123]* | c30 | [cjt]90 | c4x \ + | c8051 | clipper | craynv | csky | cydra \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | elxsi | epiphany \ + | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ + | h8300 | h8500 \ + | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i*86 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle \ + | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ + | m88110 | m88k | maxq | mb | mcore | mep | metag \ + | microblaze | microblazeel \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64eb | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mmix \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | none | np1 | ns16k | ns32k | nvptx \ + | open8 \ + | or1k* \ + | or32 \ + | orion \ + | picochip \ + | pdp10 | pdp11 | pj | pjl | pn | power \ + | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | romp | rs6000 | rx \ + | score \ + | sh | shl \ + | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ + | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | spu \ + | tahoe \ + | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tron \ + | ubicom32 \ + | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | vax \ + | visium \ + | w65 \ + | wasm32 | wasm64 \ + | we32k \ + | x86 | x86_64 | xc16x | xgate | xps100 \ + | xstormy16 | xtensa* \ + | ymp \ + | z8k | z80) + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$cpu-$vendor"\' not recognized 1>&2 + exit 1 + ;; + esac ;; esac # Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` +case $vendor in + digital*) + vendor=dec ;; - *-commodore*) - basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` + commodore*) + vendor=cbm ;; *) ;; @@ -1334,199 +1275,243 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x"$os" != x"" ] +if [ x$os != x ] then case $os in # First match some system type aliases that might get confused # with valid system types. - # -solaris* is a basic system type, with this one exception. - -auroraux) - os=-auroraux + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux ;; - -solaris1 | -solaris1.*) + bluegene*) + os=cnk + ;; + solaris1 | solaris1.*) os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; - -solaris) - os=-solaris2 + solaris) + os=solaris2 ;; - -unixware*) - os=-sysv4.2uw + unixware*) + os=sysv4.2uw ;; - -gnu/linux*) + gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; # es1800 is here to avoid being matched by es* (a different OS) - -es1800*) - os=-ose + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* | -plan9* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* | -cloudabi* | -sortix* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* | -hcos* \ - | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ - | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-musl* | -linux-uclibc* \ - | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ - | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -bme* \ - | -midnightbsd*) + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* | twizzler* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* | os108* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ + | nsk* | powerunix) # Remember, each alternative MUST END IN *, to match a version number. ;; - -qnx*) - case $basic_machine in - x86-* | i*86-*) + qnx*) + case $cpu in + x86 | i*86) ;; *) - os=-nto$os + os=nto-$os ;; esac ;; - -nto-qnx*) + hiux*) + os=hiuxwe2 ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` + nto-qnx*) ;; - -sim | -xray | -os68k* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; - -mac*) - os=`echo "$os" | sed -e 's|mac|macos|'` + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) ;; - -linux-dietlibc) - os=-linux-dietlibc + linux-dietlibc) + os=linux-dietlibc ;; - -linux*) + linux*) os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; - -sunos5*) - os=`echo "$os" | sed -e 's|sunos5|solaris2|'` + lynx*178) + os=lynxos178 ;; - -sunos6*) - os=`echo "$os" | sed -e 's|sunos6|solaris3|'` + lynx*5) + os=lynxos5 + ;; + lynx*) + os=lynxos ;; - -opened*) - os=-openedition + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` ;; - -os400*) - os=-os400 + opened*) + os=openedition ;; - -wince*) - os=-wince + os400*) + os=os400 ;; - -utek*) - os=-bsd + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; - -dynix*) - os=-bsd + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; - -acis*) - os=-aos + wince*) + os=wince ;; - -atheos*) - os=-atheos + utek*) + os=bsd ;; - -syllable*) - os=-syllable + dynix*) + os=bsd ;; - -386bsd) - os=-bsd + acis*) + os=aos ;; - -ctix* | -uts*) - os=-sysv + atheos*) + os=atheos ;; - -nova*) - os=-rtmk-nova + syllable*) + os=syllable ;; - -ns2) - os=-nextstep2 + 386bsd) + os=bsd ;; - -nsk*) - os=-nsk + ctix* | uts*) + os=sysv + ;; + nova*) + os=rtmk-nova + ;; + ns2) + os=nextstep2 ;; # Preserve the version number of sinix5. - -sinix5.*) + sinix5.*) os=`echo $os | sed -e 's|sinix|sysv|'` ;; - -sinix*) - os=-sysv4 + sinix*) + os=sysv4 ;; - -tpf*) - os=-tpf + tpf*) + os=tpf ;; - -triton*) - os=-sysv3 + triton*) + os=sysv3 ;; - -oss*) - os=-sysv3 + oss*) + os=sysv3 ;; - -svr4*) - os=-sysv4 + svr4*) + os=sysv4 ;; - -svr3) - os=-sysv3 + svr3) + os=sysv3 ;; - -sysvr4) - os=-sysv4 + sysvr4) + os=sysv4 ;; - # This must come after -sysvr4. - -sysv*) + # This must come after sysvr4. + sysv*) ;; - -ose*) - os=-ose + ose*) + os=ose ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint ;; - -zvmoe) - os=-zvmoe + zvmoe) + os=zvmoe ;; - -dicos*) - os=-dicos + dicos*) + os=dicos ;; - -pikeos*) + pikeos*) # Until real need of OS specific support for # particular features comes up, bare metal # configurations are quite functional. - case $basic_machine in + case $cpu in arm*) - os=-eabi + os=eabi ;; *) - os=-elf + os=elf ;; esac ;; - -nacl*) + nacl*) ;; - -ios) + ios) ;; - -none) + none) + ;; + *-eabi) ;; *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; @@ -1543,254 +1528,261 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. -case $basic_machine in +case $cpu-$vendor in score-*) - os=-elf + os=elf ;; spu-*) - os=-elf + os=elf ;; *-acorn) - os=-riscix1.2 + os=riscix1.2 ;; arm*-rebel) - os=-linux + os=linux ;; arm*-semi) - os=-aout + os=aout ;; c4x-* | tic4x-*) - os=-coff + os=coff ;; c8051-*) - os=-elf + os=elf + ;; + clipper-intergraph) + os=clix ;; hexagon-*) - os=-elf + os=elf ;; tic54x-*) - os=-coff + os=coff ;; tic55x-*) - os=-coff + os=coff ;; tic6x-*) - os=-coff + os=coff ;; # This must come before the *-dec entry. pdp10-*) - os=-tops20 + os=tops20 ;; pdp11-*) - os=-none + os=none ;; *-dec | vax-*) - os=-ultrix4.2 + os=ultrix4.2 ;; m68*-apollo) - os=-domain + os=domain ;; i386-sun) - os=-sunos4.0.2 + os=sunos4.0.2 ;; m68000-sun) - os=-sunos3 + os=sunos3 ;; m68*-cisco) - os=-aout + os=aout ;; mep-*) - os=-elf + os=elf ;; mips*-cisco) - os=-elf + os=elf ;; mips*-*) - os=-elf + os=elf ;; or32-*) - os=-coff + os=coff ;; *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 + os=sysv3 ;; sparc-* | *-sun) - os=-sunos4.1.1 + os=sunos4.1.1 ;; pru-*) - os=-elf + os=elf ;; *-be) - os=-beos + os=beos ;; *-ibm) - os=-aix + os=aix ;; *-knuth) - os=-mmixware + os=mmixware ;; *-wec) - os=-proelf + os=proelf ;; *-winbond) - os=-proelf + os=proelf ;; *-oki) - os=-proelf + os=proelf ;; *-hp) - os=-hpux + os=hpux ;; *-hitachi) - os=-hiux + os=hiux ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv + os=sysv ;; *-cbm) - os=-amigaos + os=amigaos ;; *-dg) - os=-dgux + os=dgux ;; *-dolphin) - os=-sysv3 + os=sysv3 ;; m68k-ccur) - os=-rtu + os=rtu ;; m88k-omron*) - os=-luna + os=luna ;; *-next) - os=-nextstep + os=nextstep ;; *-sequent) - os=-ptx + os=ptx ;; *-crds) - os=-unos + os=unos ;; *-ns) - os=-genix + os=genix ;; i370-*) - os=-mvs + os=mvs ;; *-gould) - os=-sysv + os=sysv ;; *-highlevel) - os=-bsd + os=bsd ;; *-encore) - os=-bsd + os=bsd ;; *-sgi) - os=-irix + os=irix ;; *-siemens) - os=-sysv4 + os=sysv4 ;; *-masscomp) - os=-rtu + os=rtu ;; f30[01]-fujitsu | f700-fujitsu) - os=-uxpv + os=uxpv ;; *-rom68k) - os=-coff + os=coff ;; *-*bug) - os=-coff + os=coff ;; *-apple) - os=-macos + os=macos ;; *-atari*) - os=-mint + os=mint + ;; + *-wrs) + os=vxworks ;; *) - os=-none + os=none ;; esac fi # Here we handle the case where we know the os, and the CPU type, but not the # manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) +case $vendor in + unknown) case $os in - -riscix*) + riscix*) vendor=acorn ;; - -sunos*) + sunos*) vendor=sun ;; - -cnk*|-aix*) + cnk*|-aix*) vendor=ibm ;; - -beos*) + beos*) vendor=be ;; - -hpux*) + hpux*) vendor=hp ;; - -mpeix*) + mpeix*) vendor=hp ;; - -hiux*) + hiux*) vendor=hitachi ;; - -unos*) + unos*) vendor=crds ;; - -dgux*) + dgux*) vendor=dg ;; - -luna*) + luna*) vendor=omron ;; - -genix*) + genix*) vendor=ns ;; - -mvs* | -opened*) + clix*) + vendor=intergraph + ;; + mvs* | opened*) vendor=ibm ;; - -os400*) + os400*) vendor=ibm ;; - -ptx*) + ptx*) vendor=sequent ;; - -tpf*) + tpf*) vendor=ibm ;; - -vxsim* | -vxworks* | -windiss*) + vxsim* | vxworks* | windiss*) vendor=wrs ;; - -aux*) + aux*) vendor=apple ;; - -hms*) + hms*) vendor=hitachi ;; - -mpw* | -macos*) + mpw* | macos*) vendor=apple ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) vendor=atari ;; - -vos*) + vos*) vendor=stratus ;; esac - basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` ;; esac -echo "$basic_machine$os" +echo "$cpu-$vendor-$os" exit # Local variables: diff --git a/dist2/configure b/dist2/configure index 1b3b0285..c22d3abe 100755 --- a/dist2/configure +++ b/dist2/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for PCRE2 10.33. +# Generated by GNU Autoconf 2.69 for PCRE2 10.36. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -587,8 +587,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='PCRE2' PACKAGE_TARNAME='pcre2' -PACKAGE_VERSION='10.33' -PACKAGE_STRING='PCRE2 10.33' +PACKAGE_VERSION='10.36' +PACKAGE_STRING='PCRE2 10.36' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -633,6 +633,8 @@ ac_subst_vars='am__EXEEXT_FALSE am__EXEEXT_TRUE LTLIBOBJS LIBOBJS +LIB_POSTFIX +CET_CFLAGS WITH_GCOV_FALSE WITH_GCOV_TRUE GCOV_LIBS @@ -1413,7 +1415,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures PCRE2 10.33 to adapt to many kinds of systems. +\`configure' configures PCRE2 10.36 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1483,7 +1485,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of PCRE2 10.33:";; + short | recursive ) echo "Configuration of PCRE2 10.36:";; esac cat <<\_ACEOF @@ -1509,6 +1511,7 @@ Optional Features: --enable-debug enable debugging code --enable-jit enable Just-In-Time compiling support --enable-jit-sealloc enable SELinux compatible execmem allocator in JIT + (experimental) --disable-pcre2grep-jit disable JIT support in pcre2grep --disable-pcre2grep-callout disable callout script support in pcre2grep @@ -1662,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -PCRE2 configure 10.33 +PCRE2 configure 10.36 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2157,7 +2160,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by PCRE2 $as_me 10.33, which was +It was created by PCRE2 $as_me 10.36, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3021,7 +3024,7 @@ fi # Define the identity of the package. PACKAGE='pcre2' - VERSION='10.33' + VERSION='10.36' cat >>confdefs.h <<_ACEOF @@ -5249,7 +5252,7 @@ esac -macro_version='2.4.6.42-b88ce' +macro_version='2.4.6.42-b88ce-dirty' macro_revision='2.4.6.42' @@ -12978,12 +12981,56 @@ _ACEOF +# Check for Clang __attribute__((uninitialized)) feature + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __attribute__((uninitialized))" >&5 +$as_echo_n "checking for __attribute__((uninitialized))... " >&6; } +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +tmp_CFLAGS=$CFLAGS +CFLAGS="$CFLAGS -Werror" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +char buf[128] __attribute__((uninitialized));(void)buf + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + pcre2_cc_cv_attribute_uninitialized=yes +else + pcre2_cc_cv_attribute_uninitialized=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pcre2_cc_cv_attribute_uninitialized" >&5 +$as_echo "$pcre2_cc_cv_attribute_uninitialized" >&6; } +if test "$pcre2_cc_cv_attribute_uninitialized" = yes; then + +$as_echo "#define HAVE_ATTRIBUTE_UNINITIALIZED 1" >>confdefs.h + +fi +CFLAGS=$tmp_CFLAGS +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + # Versioning PCRE2_MAJOR="10" -PCRE2_MINOR="33" +PCRE2_MINOR="36" PCRE2_PRERELEASE="" -PCRE2_DATE="2019-04-16" +PCRE2_DATE="2020-12-04" if test "$PCRE2_MINOR" = "08" -o "$PCRE2_MINOR" = "09" then @@ -13111,14 +13158,21 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext echo checking for JIT support on this hardware... $enable_jit fi -# Handle --enable-jit-sealloc (disabled by default) -# Check whether --enable-jit-sealloc was given. +# Handle --enable-jit-sealloc (disabled by default and only experimental) +case $host_os in + linux* | netbsd*) + # Check whether --enable-jit-sealloc was given. if test "${enable_jit_sealloc+set}" = set; then : enableval=$enable_jit_sealloc; else enable_jit_sealloc=no fi + ;; + *) + enable_jit_sealloc=unsupported + ;; +esac # Handle --disable-pcre2grep-jit (enabled by default) # Check whether --enable-pcre2grep-jit was given. @@ -13813,7 +13867,7 @@ fi # Checks for library functions. -for ac_func in bcopy memmove strerror mkostemp secure_getenv +for ac_func in bcopy memfd_create memmove mkostemp secure_getenv strerror do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -14972,16 +15026,16 @@ esac # are m4 variables, assigned above. EXTRA_LIBPCRE2_8_LDFLAGS="$EXTRA_LIBPCRE2_8_LDFLAGS \ - $NO_UNDEFINED -version-info 8:0:8" + $NO_UNDEFINED -version-info 10:1:10" EXTRA_LIBPCRE2_16_LDFLAGS="$EXTRA_LIBPCRE2_16_LDFLAGS \ - $NO_UNDEFINED -version-info 8:0:8" + $NO_UNDEFINED -version-info 10:1:10" EXTRA_LIBPCRE2_32_LDFLAGS="$EXTRA_LIBPCRE2_32_LDFLAGS \ - $NO_UNDEFINED -version-info 8:0:8" + $NO_UNDEFINED -version-info 10:1:10" EXTRA_LIBPCRE2_POSIX_LDFLAGS="$EXTRA_LIBPCRE2_POSIX_LDFLAGS \ - $NO_UNDEFINED -version-info 2:2:0" + $NO_UNDEFINED -version-info 2:3:0" @@ -15441,7 +15495,52 @@ else fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Intel CET is enabled" >&5 +$as_echo_n "checking whether Intel CET is enabled... " >&6; } +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +#ifndef __CET__ +# error CET is not enabled +#endif + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + pcre2_cc_cv_intel_cet_enabled=yes +else + pcre2_cc_cv_intel_cet_enabled=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pcre2_cc_cv_intel_cet_enabled" >&5 +$as_echo "$pcre2_cc_cv_intel_cet_enabled" >&6; } +if test "$pcre2_cc_cv_intel_cet_enabled" = yes; then + CET_CFLAGS="-mshstk" + +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +# LIB_POSTFIX is used by CMakeLists.txt for Windows debug builds. +# Pass empty LIB_POSTFIX to *.pc files and pcre2-config here. + + # Produce these files, in addition to config.h. + ac_config_files="$ac_config_files Makefile libpcre2-8.pc libpcre2-16.pc libpcre2-32.pc libpcre2-posix.pc pcre2-config src/pcre2.h" @@ -16024,7 +16123,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by PCRE2 $as_me 10.33, which was +This file was extended by PCRE2 $as_me 10.36, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16090,7 +16189,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -PCRE2 config.status 10.33 +PCRE2 config.status 10.36 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -17193,7 +17292,9 @@ $as_echo X/"$am_mf" | { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} as_fn_error $? "Something went wrong bootstrapping makefile fragments - for automatic dependency tracking. Try re-running configure with the + for automatic dependency tracking. If GNU make was not used, consider + re-running the configure script with MAKE=\"gmake\" (or whatever is + necessary). You can also try re-running configure with the '--disable-dependency-tracking' option to at least be able to build the package (albeit without support for automatic dependency tracking). See \`config.log' for more details" "$LINENO" 5; } @@ -17220,7 +17321,6 @@ See \`config.log' for more details" "$LINENO" 5; } cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff --git a/dist2/configure.ac b/dist2/configure.ac index f3086f18..af26f0b8 100644 --- a/dist2/configure.ac +++ b/dist2/configure.ac @@ -9,19 +9,19 @@ dnl The PCRE2_PRERELEASE feature is for identifying release candidates. It might dnl be defined as -RC2, for example. For real releases, it should be empty. m4_define(pcre2_major, [10]) -m4_define(pcre2_minor, [33]) +m4_define(pcre2_minor, [36]) m4_define(pcre2_prerelease, []) -m4_define(pcre2_date, [2019-04-16]) +m4_define(pcre2_date, [2020-12-04]) + +# Libtool shared library interface versions (current:revision:age) +m4_define(libpcre2_8_version, [10:1:10]) +m4_define(libpcre2_16_version, [10:1:10]) +m4_define(libpcre2_32_version, [10:1:10]) +m4_define(libpcre2_posix_version, [2:3:0]) # NOTE: The CMakeLists.txt file searches for the above variables in the first # 50 lines of this file. Please update that if the variables above are moved. -# Libtool shared library interface versions (current:revision:age) -m4_define(libpcre2_8_version, [8:0:8]) -m4_define(libpcre2_16_version, [8:0:8]) -m4_define(libpcre2_32_version, [8:0:8]) -m4_define(libpcre2_posix_version, [2:2:0]) - AC_PREREQ(2.57) AC_INIT(PCRE2, pcre2_major.pcre2_minor[]pcre2_prerelease, , pcre2) AC_CONFIG_SRCDIR([src/pcre2.h.in]) @@ -176,11 +176,18 @@ if test "$enable_jit" = "auto"; then echo checking for JIT support on this hardware... $enable_jit fi -# Handle --enable-jit-sealloc (disabled by default) -AC_ARG_ENABLE(jit-sealloc, - AS_HELP_STRING([--enable-jit-sealloc], - [enable SELinux compatible execmem allocator in JIT]), - , enable_jit_sealloc=no) +# Handle --enable-jit-sealloc (disabled by default and only experimental) +case $host_os in + linux* | netbsd*) + AC_ARG_ENABLE(jit-sealloc, + AS_HELP_STRING([--enable-jit-sealloc], + [enable SELinux compatible execmem allocator in JIT (experimental)]), + ,enable_jit_sealloc=no) + ;; + *) + enable_jit_sealloc=unsupported + ;; +esac # Handle --disable-pcre2grep-jit (enabled by default) AC_ARG_ENABLE(pcre2grep-jit, @@ -507,7 +514,7 @@ AC_TYPE_SIZE_T # Checks for library functions. -AC_CHECK_FUNCS(bcopy memmove strerror mkostemp secure_getenv) +AC_CHECK_FUNCS(bcopy memfd_create memmove mkostemp secure_getenv strerror) # Check for the availability of libz (aka zlib) @@ -999,7 +1006,27 @@ fi # enable_coverage AM_CONDITIONAL([WITH_GCOV],[test "x$enable_coverage" = "xyes"]) +AC_MSG_CHECKING([whether Intel CET is enabled]) +AC_LANG_PUSH([C]) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(, + [[#ifndef __CET__ +# error CET is not enabled +#endif]])], + [pcre2_cc_cv_intel_cet_enabled=yes], + [pcre2_cc_cv_intel_cet_enabled=no]) +AC_MSG_RESULT([$pcre2_cc_cv_intel_cet_enabled]) +if test "$pcre2_cc_cv_intel_cet_enabled" = yes; then + CET_CFLAGS="-mshstk" + AC_SUBST([CET_CFLAGS]) +fi +AC_LANG_POP([C]) + +# LIB_POSTFIX is used by CMakeLists.txt for Windows debug builds. +# Pass empty LIB_POSTFIX to *.pc files and pcre2-config here. +AC_SUBST(LIB_POSTFIX) + # Produce these files, in addition to config.h. + AC_CONFIG_FILES( Makefile libpcre2-8.pc diff --git a/dist2/depcomp b/dist2/depcomp index 65cbf709..6b391623 100755 --- a/dist2/depcomp +++ b/dist2/depcomp @@ -3,7 +3,7 @@ scriptversion=2018-03-07.03; # UTC -# Copyright (C) 1999-2018 Free Software Foundation, Inc. +# Copyright (C) 1999-2020 Free Software Foundation, Inc. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/dist2/doc/html/NON-AUTOTOOLS-BUILD.txt b/dist2/doc/html/NON-AUTOTOOLS-BUILD.txt index 39e7620a..a73c058b 100644 --- a/dist2/doc/html/NON-AUTOTOOLS-BUILD.txt +++ b/dist2/doc/html/NON-AUTOTOOLS-BUILD.txt @@ -74,14 +74,14 @@ can skip ahead to the CMake section. src/pcre2_chartables.c. OR: - Compile src/dftables.c as a stand-alone program (using -DHAVE_CONFIG_H - if you have set up src/config.h), and then run it with the single - argument "src/pcre2_chartables.c". This generates a set of standard - character tables and writes them to that file. The tables are generated - using the default C locale for your system. If you want to use a locale - that is specified by LC_xxx environment variables, add the -L option to - the dftables command. You must use this method if you are building on a - system that uses EBCDIC code. + Compile src/pcre2_dftables.c as a stand-alone program (using + -DHAVE_CONFIG_H if you have set up src/config.h), and then run it with + the single argument "src/pcre2_chartables.c". This generates a set of + standard character tables and writes them to that file. The tables are + generated using the default C locale for your system. If you want to use + a locale that is specified by LC_xxx environment variables, add the -L + option to the pcre2_dftables command. You must use this method if you + are building on a system that uses EBCDIC code. The tables in src/pcre2_chartables.c are defaults. The caller of PCRE2 can specify alternative tables at run time. diff --git a/dist2/doc/html/README.txt b/dist2/doc/html/README.txt index ff9a6af6..1d6df8f2 100644 --- a/dist2/doc/html/README.txt +++ b/dist2/doc/html/README.txt @@ -5,11 +5,11 @@ PCRE2 is a re-working of the original PCRE1 library to provide an entirely new API. Since its initial release in 2015, there has been further development of the code and it now differs from PCRE1 in more than just the API. There are new features and the internals have been improved. The latest release of PCRE2 is -always available in three alternative formats from: +available in three alternative formats from: - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.tar.gz - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.tar.bz2 - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-xxx.zip +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.gz +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.bz2 +https://ftp.pcre.org/pub/pcre/pcre2-10.xx.tar.zip There is a mailing list for discussion about the development of PCRE (both the original and new APIs) at pcre-dev@exim.org. You can access the archives and @@ -164,9 +164,11 @@ library. They are also documented in the pcre2build man page. will be a compile time error. If in doubt, use --enable-jit=auto, which enables JIT only if the current hardware is supported. -. If you are enabling JIT under SELinux you may also want to add - --enable-jit-sealloc, which enables the use of an execmem allocator in JIT - that is compatible with SELinux. This has no effect if JIT is not enabled. +. If you are enabling JIT under SELinux environment you may also want to add + --enable-jit-sealloc, which enables the use of an executable memory allocator + that is compatible with SELinux. Warning: this allocator is experimental! + It does not support fork() operation and may crash when no disk space is + available. This option has no effect if JIT is disabled. . If you do not want to make use of the default support for UTF-8 Unicode character strings in the 8-bit library, UTF-16 Unicode character strings in @@ -267,9 +269,9 @@ library. They are also documented in the pcre2build man page. --enable-rebuild-chartables - a program called dftables is compiled and run in the default C locale when - you obey "make". It builds a source file called pcre2_chartables.c. If you do - not specify this option, pcre2_chartables.c is created as a copy of + a program called pcre2_dftables is compiled and run in the default C locale + when you obey "make". It builds a source file called pcre2_chartables.c. If + you do not specify this option, pcre2_chartables.c is created as a copy of pcre2_chartables.c.dist. See "Character tables" below for further information. @@ -295,8 +297,8 @@ library. They are also documented in the pcre2build man page. unaddressable. This allows it to detect invalid memory accesses, and is mostly useful for debugging PCRE2 itself. -. In environments where the gcc compiler is used and lcov version 1.6 or above - is installed, if you specify +. In environments where the gcc compiler is used and lcov is installed, if you + specify --enable-coverage @@ -546,11 +548,11 @@ Cross-compiling using autotools You can specify CC and CFLAGS in the normal way to the "configure" command, in order to cross-compile PCRE2 for some other host. However, you should NOT -specify --enable-rebuild-chartables, because if you do, the dftables.c source -file is compiled and run on the local host, in order to generate the inbuilt -character tables (the pcre2_chartables.c file). This will probably not work, -because dftables.c needs to be compiled with the local compiler, not the cross -compiler. +specify --enable-rebuild-chartables, because if you do, the pcre2_dftables.c +source file is compiled and run on the local host, in order to generate the +inbuilt character tables (the pcre2_chartables.c file). This will probably not +work, because pcre2_dftables.c needs to be compiled with the local compiler, +not the cross compiler. When --enable-rebuild-chartables is not specified, pcre2_chartables.c is created by making a copy of pcre2_chartables.c.dist, which is a default set of @@ -558,9 +560,10 @@ tables that assumes ASCII code. Cross-compiling with the default tables should not be a problem. If you need to modify the character tables when cross-compiling, you should -move pcre2_chartables.c.dist out of the way, then compile dftables.c by hand -and run it on the local host to make a new version of pcre2_chartables.c.dist. -Then when you cross-compile PCRE2 this new version of the tables will be used. +move pcre2_chartables.c.dist out of the way, then compile pcre2_dftables.c by +hand and run it on the local host to make a new version of +pcre2_chartables.c.dist. See the pcre2build section "Creating character tables +at build time" for more details. Making new tarballs @@ -719,8 +722,8 @@ compile context. The source file called pcre2_chartables.c contains the default set of tables. By default, this is created as a copy of pcre2_chartables.c.dist, which contains tables for ASCII coding. However, if --enable-rebuild-chartables is -specified for ./configure, a different version of pcre2_chartables.c is built -by the program dftables (compiled from dftables.c), which uses the ANSI C +specified for ./configure, a new version of pcre2_chartables.c is built by the +program pcre2_dftables (compiled from pcre2_dftables.c), which uses the ANSI C character handling functions such as isalnum(), isalpha(), isupper(), islower(), etc. to build the table sources. This means that the default C locale that is set for your system will control the contents of these default @@ -730,32 +733,40 @@ file does not get automatically re-generated. The best way to do this is to move pcre2_chartables.c.dist out of the way and replace it with your customized tables. -When the dftables program is run as a result of --enable-rebuild-chartables, -it uses the default C locale that is set on your system. It does not pay -attention to the LC_xxx environment variables. In other words, it uses the -system's default locale rather than whatever the compiling user happens to have -set. If you really do want to build a source set of character tables in a -locale that is specified by the LC_xxx variables, you can run the dftables -program by hand with the -L option. For example: +When the pcre2_dftables program is run as a result of specifying +--enable-rebuild-chartables, it uses the default C locale that is set on your +system. It does not pay attention to the LC_xxx environment variables. In other +words, it uses the system's default locale rather than whatever the compiling +user happens to have set. If you really do want to build a source set of +character tables in a locale that is specified by the LC_xxx variables, you can +run the pcre2_dftables program by hand with the -L option. For example: - ./dftables -L pcre2_chartables.c.special + ./pcre2_dftables -L pcre2_chartables.c.special -The first two 256-byte tables provide lower casing and case flipping functions, -respectively. The next table consists of three 32-byte bit maps which identify -digits, "word" characters, and white space, respectively. These are used when -building 32-byte bit maps that represent character classes for code points less -than 256. The final 256-byte table has bits indicating various character types, -as follows: +The second argument names the file where the source code for the tables is +written. The first two 256-byte tables provide lower casing and case flipping +functions, respectively. The next table consists of a number of 32-byte bit +maps which identify certain character classes such as digits, "word" +characters, white space, etc. These are used when building 32-byte bit maps +that represent character classes for code points less than 256. The final +256-byte table has bits indicating various character types, as follows: 1 white space character 2 letter - 4 decimal digit - 8 hexadecimal digit + 4 lower case letter + 8 decimal digit 16 alphanumeric or '_' - 128 regular expression metacharacter or binary zero -You should not alter the set of characters that contain the 128 bit, as that -will cause PCRE2 to malfunction. +You can also specify -b (with or without -L) when running pcre2_dftables. This +causes the tables to be written in binary instead of as source code. A set of +binary tables can be loaded into memory by an application and passed to +pcre2_compile() in the same way as tables created dynamically by calling +pcre2_maketables(). The tables are just a string of bytes, independent of +hardware characteristics such as endianness. This means they can be bundled +with an application that runs in different environments, to ensure consistent +behaviour. + +See also the pcre2build section "Creating character tables at build time". File manifest @@ -766,7 +777,7 @@ The distribution should contain the files listed below. (A) Source files for the PCRE2 library functions and their headers are found in the src directory: - src/dftables.c auxiliary program for building pcre2_chartables.c + src/pcre2_dftables.c auxiliary program for building pcre2_chartables.c when --enable-rebuild-chartables is specified src/pcre2_chartables.c.dist a default set of character tables that assume @@ -890,6 +901,6 @@ The distribution should contain the files listed below. ) environments Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk -Last updated: 16 April 2019 +Email local part: Philip.Hazel +Email domain: gmail.com +Last updated: 04 December 2020 diff --git a/dist2/doc/html/index.html b/dist2/doc/html/index.html index 82b43c35..2c7c5fb2 100644 --- a/dist2/doc/html/index.html +++ b/dist2/doc/html/index.html @@ -146,6 +146,9 @@ in the library. <tr><td><a href="pcre2_get_mark.html">pcre2_get_mark</a></td> <td> Get a (*MARK) name</td></tr> +<tr><td><a href="pcre2_get_match_data_size.html">pcre2_get_match_data_size</a></td> + <td> Get the size of a match data block</td></tr> + <tr><td><a href="pcre2_get_ovector_count.html">pcre2_get_ovector_count</a></td> <td> Get the ovector count</td></tr> @@ -176,6 +179,9 @@ in the library. <tr><td><a href="pcre2_maketables.html">pcre2_maketables</a></td> <td> Build character tables in current locale</td></tr> +<tr><td><a href="pcre2_maketables_free.html">pcre2_maketables_free</a></td> + <td> Free character tables</td></tr> + <tr><td><a href="pcre2_match.html">pcre2_match</a></td> <td> Match a compiled pattern to a subject string (Perl compatible)</td></tr> diff --git a/dist2/doc/html/pcre2_compile.html b/dist2/doc/html/pcre2_compile.html index 23f75e10..f6485f22 100644 --- a/dist2/doc/html/pcre2_compile.html +++ b/dist2/doc/html/pcre2_compile.html @@ -65,6 +65,7 @@ The option bits are: PCRE2_EXTENDED Ignore white space and # comments PCRE2_FIRSTLINE Force matching to be before newline PCRE2_LITERAL Pattern characters are all literal + PCRE2_MATCH_INVALID_UTF Enable support for matching invalid UTF PCRE2_MATCH_UNSET_BACKREF Match unset backreferences PCRE2_MULTILINE ^ and $ match newlines within data PCRE2_NEVER_BACKSLASH_C Lock out the use of \C in patterns diff --git a/dist2/doc/html/pcre2_get_match_data_size.html b/dist2/doc/html/pcre2_get_match_data_size.html new file mode 100644 index 00000000..113ecaab --- /dev/null +++ b/dist2/doc/html/pcre2_get_match_data_size.html @@ -0,0 +1,39 @@ +<html> +<head> +<title>pcre2_get_match_data_size specification</title> +</head> +<body bgcolor="#FFFFFF" text="#00005A" link="#0066FF" alink="#3399FF" vlink="#2222BB"> +<h1>pcre2_get_match_data_size man page</h1> +<p> +Return to the <a href="index.html">PCRE2 index page</a>. +</p> +<p> +This page is part of the PCRE2 HTML documentation. It was generated +automatically from the original man page. If there is any nonsense in it, +please consult the man page, in case the conversion went wrong. +<br> +<br><b> +SYNOPSIS +</b><br> +<P> +<b>#include <pcre2.h></b> +</P> +<P> +<b>PCRE2_SIZE pcre2_get_match_data_size(pcre2_match_data *<i>match_data</i>);</b> +</P> +<br><b> +DESCRIPTION +</b><br> +<P> +This function returns the size, in bytes, of the match data block that is its +argument. +</P> +<P> +There is a complete description of the PCRE2 native API in the +<a href="pcre2api.html"><b>pcre2api</b></a> +page and a description of the POSIX API in the +<a href="pcre2posix.html"><b>pcre2posix</b></a> +page. +<p> +Return to the <a href="index.html">PCRE2 index page</a>. +</p> diff --git a/dist2/doc/html/pcre2_jit_compile.html b/dist2/doc/html/pcre2_jit_compile.html index bbaa82c2..873d0dde 100644 --- a/dist2/doc/html/pcre2_jit_compile.html +++ b/dist2/doc/html/pcre2_jit_compile.html @@ -40,11 +40,17 @@ bits: PCRE2_JIT_COMPLETE compile code for full matching PCRE2_JIT_PARTIAL_SOFT compile code for soft partial matching PCRE2_JIT_PARTIAL_HARD compile code for hard partial matching - PCRE2_JIT_INVALID_UTF compile code to handle invalid UTF </pre> +There is also an obsolete option called PCRE2_JIT_INVALID_UTF, which has been +superseded by the <b>pcre2_compile()</b> option PCRE2_MATCH_INVALID_UTF. The old +option is deprecated and may be removed in the future. +</P> +<P> The yield of the function is 0 for success, or a negative error code otherwise. In particular, PCRE2_ERROR_JIT_BADOPTION is returned if JIT is not supported or -if an unknown bit is set in <i>options</i>. +if an unknown bit is set in <i>options</i>. The function can also return +PCRE2_ERROR_NOMEMORY if JIT is unable to allocate executable memory for the +compiler, even if it was because of a system security restriction. </P> <P> There is a complete description of the PCRE2 native API in the diff --git a/dist2/doc/html/pcre2_jit_free_unused_memory.html b/dist2/doc/html/pcre2_jit_free_unused_memory.html index 8b59b8e1..7f37e583 100644 --- a/dist2/doc/html/pcre2_jit_free_unused_memory.html +++ b/dist2/doc/html/pcre2_jit_free_unused_memory.html @@ -29,7 +29,7 @@ This function frees unused JIT executable memory. The argument is a general context, for custom memory management, or NULL for standard memory management. JIT memory allocation retains some memory in order to improve future JIT compilation speed. In low memory conditions, -\fBpcre2_jit_free_unused_memory()\fB can be used to cause this memory to be +<b>pcre2_jit_free_unused_memory()</b> can be used to cause this memory to be freed. </P> <P> diff --git a/dist2/doc/html/pcre2_jit_match.html b/dist2/doc/html/pcre2_jit_match.html index 1d59667c..8629e4a4 100644 --- a/dist2/doc/html/pcre2_jit_match.html +++ b/dist2/doc/html/pcre2_jit_match.html @@ -33,7 +33,9 @@ processed by the JIT compiler against a given subject string, using a matching algorithm that is similar to Perl's. It is a "fast path" interface to JIT, and it bypasses some of the sanity checks that <b>pcre2_match()</b> applies. Its arguments are exactly the same as for -<a href="pcre2_match.html"><b>pcre2_match()</b>.</a> +<a href="pcre2_match.html"><b>pcre2_match()</b>,</a> +except that the subject string must be specified with a length; +PCRE2_ZERO_TERMINATED is not supported. </P> <P> The supported options are PCRE2_NOTBOL, PCRE2_NOTEOL, PCRE2_NOTEMPTY, diff --git a/dist2/doc/html/pcre2_maketables.html b/dist2/doc/html/pcre2_maketables.html index 6d240e32..19636545 100644 --- a/dist2/doc/html/pcre2_maketables.html +++ b/dist2/doc/html/pcre2_maketables.html @@ -19,7 +19,7 @@ SYNOPSIS <b>#include <pcre2.h></b> </P> <P> -<b>const unsigned char *pcre2_maketables(pcre2_general_context *<i>gcontext</i>);</b> +<b>const uint8_t *pcre2_maketables(pcre2_general_context *<i>gcontext</i>);</b> </P> <br><b> DESCRIPTION diff --git a/dist2/doc/html/pcre2_maketables_free.html b/dist2/doc/html/pcre2_maketables_free.html new file mode 100644 index 00000000..7316ab25 --- /dev/null +++ b/dist2/doc/html/pcre2_maketables_free.html @@ -0,0 +1,44 @@ +<html> +<head> +<title>pcre2_maketables_free specification</title> +</head> +<body bgcolor="#FFFFFF" text="#00005A" link="#0066FF" alink="#3399FF" vlink="#2222BB"> +<h1>pcre2_maketables_free man page</h1> +<p> +Return to the <a href="index.html">PCRE2 index page</a>. +</p> +<p> +This page is part of the PCRE2 HTML documentation. It was generated +automatically from the original man page. If there is any nonsense in it, +please consult the man page, in case the conversion went wrong. +<br> +<br><b> +SYNOPSIS +</b><br> +<P> +<b>#include <pcre2.h></b> +</P> +<P> +<b>void pcre2_maketables_free(pcre2_general_context *<i>gcontext</i>,</b> +<b> const uint8_t *<i>tables</i>);</b> +</P> +<br><b> +DESCRIPTION +</b><br> +<P> +This function discards a set of character tables that were created by a call +to +<a href="pcre2_maketables.html"><b>pcre2_maketables()</b>.</a> +</P> +<P> +The <i>gcontext</i> parameter should match what was used in that call to +account for any custom allocators that might be in use; if it is NULL +the system <b>free()</b> is used. +</P> +<P> +There is a complete description of the PCRE2 native API in the +<a href="pcre2api.html"><b>pcre2api</b></a> +page. +<p> +Return to the <a href="index.html">PCRE2 index page</a>. +</p> diff --git a/dist2/doc/html/pcre2_set_character_tables.html b/dist2/doc/html/pcre2_set_character_tables.html index 5a1bd20e..8564eea6 100644 --- a/dist2/doc/html/pcre2_set_character_tables.html +++ b/dist2/doc/html/pcre2_set_character_tables.html @@ -20,16 +20,19 @@ SYNOPSIS </P> <P> <b>int pcre2_set_character_tables(pcre2_compile_context *<i>ccontext</i>,</b> -<b> const unsigned char *<i>tables</i>);</b> +<b> const uint8_t *<i>tables</i>);</b> </P> <br><b> DESCRIPTION </b><br> <P> This function sets a pointer to custom character tables within a compile -context. The second argument must be the result of a call to -<b>pcre2_maketables()</b> or NULL to request the default tables. The result is -always zero. +context. The second argument must point to a set of PCRE2 character tables or +be NULL to request the default tables. The result is always zero. Character +tables can be created by calling <b>pcre2_maketables()</b> or by running the +<b>pcre2_dftables</b> maintenance command in binary mode (see the +<a href="pcre2build.html"><b>pcre2build</b></a> +documentation). </P> <P> There is a complete description of the PCRE2 native API in the diff --git a/dist2/doc/html/pcre2_substitute.html b/dist2/doc/html/pcre2_substitute.html index 2215ce9d..10b2267e 100644 --- a/dist2/doc/html/pcre2_substitute.html +++ b/dist2/doc/html/pcre2_substitute.html @@ -48,8 +48,8 @@ Its arguments are: <i>outlengthptr</i> Points to the length of the output buffer </pre> A match data block is needed only if you want to inspect the data from the -match that is returned in that block. A match context is needed only if you -want to: +final match that is returned in that block or if PCRE2_SUBSTITUTE_MATCHED is +set. A match context is needed only if you want to: <pre> Set up a callout function Set a matching offset limit @@ -57,9 +57,14 @@ want to: Change the backtracking depth limit Set custom memory management in the match context </pre> -The <i>length</i>, <i>startoffset</i> and <i>rlength</i> values are code -units, not characters, as is the contents of the variable pointed at by -<i>outlengthptr</i>, which is updated to the actual length of the new string. +The <i>length</i>, <i>startoffset</i> and <i>rlength</i> values are code units, +not characters, as is the contents of the variable pointed at by +<i>outlengthptr</i>. This variable must contain the length of the output buffer +when the function is called. If the function is successful, the value is +changed to the length of the new string, excluding the trailing zero that is +automatically added. +</P> +<P> The subject and replacement lengths can be given as PCRE2_ZERO_TERMINATED for zero-terminated strings. The options are: <pre> @@ -74,12 +79,24 @@ zero-terminated strings. The options are: PCRE2_UTF was set at compile time) PCRE2_SUBSTITUTE_EXTENDED Do extended replacement processing PCRE2_SUBSTITUTE_GLOBAL Replace all occurrences in the subject + PCRE2_SUBSTITUTE_LITERAL The replacement string is literal + PCRE2_SUBSTITUTE_MATCHED Use pre-existing match data for 1st match PCRE2_SUBSTITUTE_OVERFLOW_LENGTH If overflow, compute needed length + PCRE2_SUBSTITUTE_REPLACEMENT_ONLY Return only replacement string(s) PCRE2_SUBSTITUTE_UNKNOWN_UNSET Treat unknown group as unset PCRE2_SUBSTITUTE_UNSET_EMPTY Simple unset insert = empty string </pre> +If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_EXTENDED, +PCRE2_SUBSTITUTE_UNKNOWN_UNSET, and PCRE2_SUBSTITUTE_UNSET_EMPTY are ignored. +</P> +<P> +If PCRE2_SUBSTITUTE_MATCHED is set, <i>match_data</i> must be non-zero; its +contents must be the result of a call to <b>pcre2_match()</b> using the same +pattern and subject. +</P> +<P> The function returns the number of substitutions, which may be zero if there -were no matches. The result can be greater than one only when +are no matches. The result may be greater than one only when PCRE2_SUBSTITUTE_GLOBAL is set. In the event of an error, a negative error code is returned. </P> diff --git a/dist2/doc/html/pcre2api.html b/dist2/doc/html/pcre2api.html index 7ca39f51..4ca0eb0c 100644 --- a/dist2/doc/html/pcre2api.html +++ b/dist2/doc/html/pcre2api.html @@ -142,7 +142,7 @@ document for an overview of all the PCRE2 documentation. <br> <br> <b>int pcre2_set_character_tables(pcre2_compile_context *<i>ccontext</i>,</b> -<b> const unsigned char *<i>tables</i>);</b> +<b> const uint8_t *<i>tables</i>);</b> <br> <br> <b>int pcre2_set_compile_extra_options(pcre2_compile_context *<i>ccontext</i>,</b> @@ -252,7 +252,7 @@ document for an overview of all the PCRE2 documentation. <b>int pcre2_substitute(const pcre2_code *<i>code</i>, PCRE2_SPTR <i>subject</i>,</b> <b> PCRE2_SIZE <i>length</i>, PCRE2_SIZE <i>startoffset</i>,</b> <b> uint32_t <i>options</i>, pcre2_match_data *<i>match_data</i>,</b> -<b> pcre2_match_context *<i>mcontext</i>, PCRE2_SPTR \fIreplacementzfP,</b> +<b> pcre2_match_context *<i>mcontext</i>, PCRE2_SPTR <i>replacementz</i>,</b> <b> PCRE2_SIZE <i>rlength</i>, PCRE2_UCHAR *<i>outputbuffer</i>,</b> <b> PCRE2_SIZE *<i>outlengthptr</i>);</b> </P> @@ -309,7 +309,11 @@ document for an overview of all the PCRE2 documentation. <b> PCRE2_SIZE <i>bufflen</i>);</b> <br> <br> -<b>const unsigned char *pcre2_maketables(pcre2_general_context *<i>gcontext</i>);</b> +<b>const uint8_t *pcre2_maketables(pcre2_general_context *<i>gcontext</i>);</b> +<br> +<br> +<b>void pcre2_maketables_free(pcre2_general_context *<i>gcontext</i>,</b> +<b> const uint8_t *<i>tables</i>);</b> <br> <br> <b>int pcre2_pattern_info(const pcre2_code *<i>code</i>, uint32_t <i>what</i>,</b> @@ -622,14 +626,15 @@ documentation for more details. <P> In a more complicated situation, where patterns are compiled only when they are first needed, but are still shared between threads, pointers to compiled -patterns must be protected from simultaneous writing by multiple threads, at -least until a pattern has been compiled. The logic can be something like this: +patterns must be protected from simultaneous writing by multiple threads. This +is somewhat tricky to do correctly. If you know that writing to a pointer is +atomic in your environment, you can use logic like this: <pre> Get a read-only (shared) lock (mutex) for pointer if (pointer == NULL) { Get a write (unique) lock for pointer - pointer = pcre2_compile(... + if (pointer == NULL) pointer = pcre2_compile(... } Release the lock Use pointer in pcre2_match() @@ -637,10 +642,39 @@ least until a pattern has been compiled. The logic can be something like this: Of course, testing for compilation errors should also be included in the code. </P> <P> -If JIT is being used, but the JIT compilation is not being done immediately, -(perhaps waiting to see if the pattern is used often enough) similar logic is -required. JIT compilation updates a pointer within the compiled code block, so -a thread must gain unique write access to the pointer before calling +The reason for checking the pointer a second time is as follows: Several +threads may have acquired the shared lock and tested the pointer for being +NULL, but only one of them will be given the write lock, with the rest kept +waiting. The winning thread will compile the pattern and store the result. +After this thread releases the write lock, another thread will get it, and if +it does not retest pointer for being NULL, will recompile the pattern and +overwrite the pointer, creating a memory leak and possibly causing other +issues. +</P> +<P> +In an environment where writing to a pointer may not be atomic, the above logic +is not sufficient. The thread that is doing the compiling may be descheduled +after writing only part of the pointer, which could cause other threads to use +an invalid value. Instead of checking the pointer itself, a separate "pointer +is valid" flag (that can be updated atomically) must be used: +<pre> + Get a read-only (shared) lock (mutex) for pointer + if (!pointer_is_valid) + { + Get a write (unique) lock for pointer + if (!pointer_is_valid) + { + pointer = pcre2_compile(... + pointer_is_valid = TRUE + } + } + Release the lock + Use pointer in pcre2_match() +</pre> +If JIT is being used, but the JIT compilation is not being done immediately +(perhaps waiting to see if the pattern is used often enough), similar logic is +required. JIT compilation updates a value within the compiled code block, so a +thread must gain unique write access to the pointer before calling <b>pcre2_jit_compile()</b>. Alternatively, <b>pcre2_code_copy()</b> or <b>pcre2_code_copy_with_tables()</b> can be used to obtain a private copy of the compiled code before calling the JIT compiler. @@ -791,10 +825,10 @@ interpreted matching functions, <i>pcre2_match()</i> and <br> <br> <b>int pcre2_set_character_tables(pcre2_compile_context *<i>ccontext</i>,</b> -<b> const unsigned char *<i>tables</i>);</b> +<b> const uint8_t *<i>tables</i>);</b> <br> <br> -The value must be the result of a call to <i>pcre2_maketables()</i>, whose only +The value must be the result of a call to <b>pcre2_maketables()</b>, whose only argument is a general context. This function builds a set of character tables in the current locale. <br> @@ -1101,10 +1135,11 @@ less than the limit set by the caller of <b>pcre2_match()</b> or <b>int pcre2_config(uint32_t <i>what</i>, void *<i>where</i>);</b> </P> <P> -The function <b>pcre2_config()</b> makes it possible for a PCRE2 client to -discover which optional features have been compiled into the PCRE2 library. The +The function <b>pcre2_config()</b> makes it possible for a PCRE2 client to find +the value of certain configuration parameters and to discover which optional +features have been compiled into the PCRE2 library. The <a href="pcre2build.html"><b>pcre2build</b></a> -documentation has more details about these optional features. +documentation has more details about these features. </P> <P> The first argument for <b>pcre2_config()</b> specifies which information is @@ -1221,6 +1256,13 @@ over compilation stack usage, see <b>pcre2_set_compile_recursion_guard()</b>. This parameter is obsolete and should not be used in new code. The output is a uint32_t integer that is always set to zero. <pre> + PCRE2_CONFIG_TABLES_LENGTH +</pre> +The output is a uint32_t integer that gives the length of PCRE2's character +processing tables in bytes. For details of these tables see the +<a href="#localesupport">section on locale support</a> +below. +<pre> PCRE2_CONFIG_UNICODE_VERSION </pre> The <i>where</i> argument should point to a buffer that is at least 24 code @@ -1347,11 +1389,12 @@ and <b>pcre2_compile()</b> returns a non-NULL value. <P> There are nearly 100 positive error codes that <b>pcre2_compile()</b> may return if it finds an error in the pattern. There are also some negative error codes -that are used for invalid UTF strings. These are the same as given by -<b>pcre2_match()</b> and <b>pcre2_dfa_match()</b>, and are described in the +that are used for invalid UTF strings when validity checking is in force. These +are the same as given by <b>pcre2_match()</b> and <b>pcre2_dfa_match()</b>, and +are described in the <a href="pcre2unicode.html"><b>pcre2unicode</b></a> -page. There is no separate documentation for the positive error codes, because -the textual error messages that are obtained by calling the +documentation. There is no separate documentation for the positive error codes, +because the textual error messages that are obtained by calling the <b>pcre2_get_error_message()</b> function (see "Obtaining a textual error message" <a href="#geterrormessage">below)</a> @@ -1476,13 +1519,16 @@ documentation. </pre> If this bit is set, letters in the pattern match both upper and lower case letters in the subject. It is equivalent to Perl's /i option, and it can be -changed within a pattern by a (?i) option setting. If PCRE2_UTF is set, Unicode -properties are used for all characters with more than one other case, and for -all characters whose code points are greater than U+007F. For lower valued -characters with only one other case, a lookup table is used for speed. When -PCRE2_UTF is not set, a lookup table is used for all code points less than 256, -and higher code points (available only in 16-bit or 32-bit mode) are treated as -not having another case. +changed within a pattern by a (?i) option setting. If either PCRE2_UTF or +PCRE2_UCP is set, Unicode properties are used for all characters with more than +one other case, and for all characters whose code points are greater than +U+007F. Note that there are two ASCII characters, K and S, that, in addition to +their lower case ASCII equivalents, are case-equivalent with U+212A (Kelvin +sign) and U+017F (long S) respectively. For lower valued characters with only +one other case, a lookup table is used for speed. When neither PCRE2_UTF nor +PCRE2_UCP is set, a lookup table is used for all code points less than 256, and +higher code points (available only in 16-bit or 32-bit mode) are treated as not +having another case. <pre> PCRE2_DOLLAR_ENDONLY </pre> @@ -1615,10 +1661,18 @@ expression engine is not the most efficient way of doing it. If you are doing a lot of literal matching and are worried about efficiency, you should consider using other approaches. The only other main options that are allowed with PCRE2_LITERAL are: PCRE2_ANCHORED, PCRE2_ENDANCHORED, PCRE2_AUTO_CALLOUT, -PCRE2_CASELESS, PCRE2_FIRSTLINE, PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, -PCRE2_UTF, and PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EXTRA_MATCH_LINE -and PCRE2_EXTRA_MATCH_WORD are also supported. Any other options cause an -error. +PCRE2_CASELESS, PCRE2_FIRSTLINE, PCRE2_MATCH_INVALID_UTF, +PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, PCRE2_UTF, and +PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EXTRA_MATCH_LINE and +PCRE2_EXTRA_MATCH_WORD are also supported. Any other options cause an error. +<pre> + PCRE2_MATCH_INVALID_UTF +</pre> +This option forces PCRE2_UTF (see below) and also enables support for matching +by <b>pcre2_match()</b> in subject strings that contain invalid UTF sequences. +This facility is not supported for DFA matching. For details, see the +<a href="pcre2unicode.html"><b>pcre2unicode</b></a> +documentation. <pre> PCRE2_MATCH_UNSET_BACKREF </pre> @@ -1753,17 +1807,22 @@ subject string does not happen. The first match attempt is run starting from the overall result is "no match". </P> <P> -There are also other start-up optimizations. For example, a minimum length for -the subject may be recorded. Consider the pattern +As another start-up optimization makes use of a minimum length for a matching +subject, which is recorded when possible. Consider the pattern <pre> - (*MARK:A)(X|Y) + (*MARK:1)B(*MARK:2)(X|Y) </pre> -The minimum length for a match is one character. If the subject is "ABC", there -will be attempts to match "ABC", "BC", and "C". An attempt to match an empty -string at the end of the subject does not take place, because PCRE2 knows that -the subject is now too short, and so the (*MARK) is never encountered. In this -case, the optimization does not affect the overall match result, which is still -"no match", but it does affect the auxiliary information that is returned. +The minimum length for a match is two characters. If the subject is "XXBB", the +"starting character" optimization skips "XX", then tries to match "BB", which +is long enough. In the process, (*MARK:2) is encountered and remembered. When +the match attempt fails, the next "B" is found, but there is only one character +left, so there are no more attempts, and "no match" is returned with the "last +mark seen" set to "2". If NO_START_OPTIMIZE is set, however, matches are tried +at every possible starting position, including at the end of the subject, where +(*MARK:1) is encountered, but there is no "B", so the "last mark seen" that is +returned is "1". In this case, the optimizations do not affect the overall +match result, which is still "no match", but they do affect the auxiliary +information that is returned. <pre> PCRE2_NO_UTF_CHECK </pre> @@ -1802,16 +1861,23 @@ are not representable in UTF-16. <pre> PCRE2_UCP </pre> -This option changes the way PCRE2 processes \B, \b, \D, \d, \S, \s, \W, -\w, and some of the POSIX character classes. By default, only ASCII characters -are recognized, but if PCRE2_UCP is set, Unicode properties are used instead to -classify characters. More details are given in the section on +This option has two effects. Firstly, it change the way PCRE2 processes \B, +\b, \D, \d, \S, \s, \W, \w, and some of the POSIX character classes. By +default, only ASCII characters are recognized, but if PCRE2_UCP is set, Unicode +properties are used instead to classify characters. More details are given in +the section on <a href="pcre2pattern.html#genericchartypes">generic character types</a> in the <a href="pcre2pattern.html"><b>pcre2pattern</b></a> page. If you set PCRE2_UCP, matching one of the items it affects takes much -longer. The option is available only if PCRE2 has been compiled with Unicode -support (which is the default). +longer. +</P> +<P> +The second effect of PCRE2_UCP is to force the use of Unicode properties for +upper/lower casing operations on characters with code points greater than 127, +even when PCRE2_UTF is not set. This makes it possible, for example, to process +strings in the 16-bit UCS-2 code. This option is available only if PCRE2 has +been compiled with Unicode support (which is the default). <pre> PCRE2_UNGREEDY </pre> @@ -1968,40 +2034,57 @@ Most (but not all) patterns can be optimized by the JIT compiler. <a name="localesupport"></a></P> <br><a name="SEC22" href="#TOC1">LOCALE SUPPORT</a><br> <P> +<b>const uint8_t *pcre2_maketables(pcre2_general_context *<i>gcontext</i>);</b> +<br> +<br> +<b>void pcre2_maketables_free(pcre2_general_context *<i>gcontext</i>,</b> +<b> const uint8_t *<i>tables</i>);</b> +</P> +<P> PCRE2 handles caseless matching, and determines whether characters are letters, digits, or whatever, by reference to a set of tables, indexed by character code -point. This applies only to characters whose code points are less than 256. By -default, higher-valued code points never match escapes such as \w or \d. -However, if PCRE2 is built with Unicode support, all characters can be tested -with \p and \P, or, alternatively, the PCRE2_UCP option can be set when a -pattern is compiled; this causes \w and friends to use Unicode property -support instead of the built-in tables. +point. However, this applies only to characters whose code points are less than +256. By default, higher-valued code points never match escapes such as \w or +\d. +</P> +<P> +When PCRE2 is built with Unicode support (the default), the Unicode properties +of all characters can be tested with \p and \P, or, alternatively, the +PCRE2_UCP option can be set when a pattern is compiled; this causes \w and +friends to use Unicode property support instead of the built-in tables. +PCRE2_UCP also causes upper/lower casing operations on characters with code +points greater than 127 to use Unicode properties. These effects apply even +when PCRE2_UTF is not set. </P> <P> The use of locales with Unicode is discouraged. If you are handling characters -with code points greater than 128, you should either use Unicode support, or +with code points greater than 127, you should either use Unicode support, or use locales, but not try to mix the two. </P> <P> -PCRE2 contains an internal set of character tables that are used by default. +PCRE2 contains a built-in set of character tables that are used by default. These are sufficient for many applications. Normally, the internal tables recognize only ASCII characters. However, when PCRE2 is built, it is possible to cause the internal tables to be rebuilt in the default "C" locale of the local system, which may cause them to be different. </P> <P> -The internal tables can be overridden by tables supplied by the application +The built-in tables can be overridden by tables supplied by the application that calls PCRE2. These may be created in a different locale from the default. As more and more applications change to using Unicode, the need for this locale support is expected to die away. </P> <P> External tables are built by calling the <b>pcre2_maketables()</b> function, in -the relevant locale. The result can be passed to <b>pcre2_compile()</b> as often -as necessary, by creating a compile context and calling -<b>pcre2_set_character_tables()</b> to set the tables pointer therein. For -example, to build and use tables that are appropriate for the French locale -(where accented characters with values greater than 128 are treated as +the relevant locale. The only argument to this function is a general context, +which can be used to pass a custom memory allocator. If the argument is NULL, +the system <b>malloc()</b> is used. The result can be passed to +<b>pcre2_compile()</b> as often as necessary, by creating a compile context and +calling <b>pcre2_set_character_tables()</b> to set the tables pointer therein. +</P> +<P> +For example, to build and use tables that are appropriate for the French locale +(where accented characters with values greater than 127 are treated as letters), the following code could be used: <pre> setlocale(LC_CTYPE, "fr_FR"); @@ -2011,16 +2094,38 @@ letters), the following code could be used: re = pcre2_compile(..., ccontext); </pre> The locale name "fr_FR" is used on Linux and other Unix-like systems; if you -are using Windows, the name for the French locale is "french". It is the -caller's responsibility to ensure that the memory containing the tables remains -available for as long as it is needed. +are using Windows, the name for the French locale is "french". </P> <P> The pointer that is passed (via the compile context) to <b>pcre2_compile()</b> -is saved with the compiled pattern, and the same tables are used by -<b>pcre2_match()</b> and <b>pcre_dfa_match()</b>. Thus, for any single pattern, -compilation and matching both happen in the same locale, but different patterns -can be processed in different locales. +is saved with the compiled pattern, and the same tables are used by the +matching functions. Thus, for any single pattern, compilation and matching both +happen in the same locale, but different patterns can be processed in different +locales. +</P> +<P> +It is the caller's responsibility to ensure that the memory containing the +tables remains available while they are still in use. When they are no longer +needed, you can discard them using <b>pcre2_maketables_free()</b>, which should +pass as its first parameter the same global context that was used to create the +tables. +</P> +<br><b> +Saving locale tables +</b><br> +<P> +The tables described above are just a sequence of binary bytes, which makes +them independent of hardware characteristics such as endianness or whether the +processor is 32-bit or 64-bit. A copy of the result of <b>pcre2_maketables()</b> +can therefore be saved in a file or elsewhere and re-used later, even in a +different program or on another computer. The size of the tables (number of +bytes) must be obtained by calling <b>pcre2_config()</b> with the +PCRE2_CONFIG_TABLES_LENGTH option because <b>pcre2_maketables()</b> does not +return this value. Note that the <b>pcre2_dftables</b> program, which is part of +the PCRE2 build system, can be used stand-alone to create a file that contains +a set of binary tables. See the +<a href="pcre2build.html#createtables"><b>pcre2build</b></a> +documentation for details. <a name="infoaboutpattern"></a></P> <br><a name="SEC23" href="#TOC1">INFORMATION ABOUT A COMPILED PATTERN</a><br> <P> @@ -2029,7 +2134,7 @@ can be processed in different locales. <P> The <b>pcre2_pattern_info()</b> function returns general information about a compiled pattern. For information about callouts, see the -<a href="pcre2pattern.html#infoaboutcallouts">next section.</a> +<a href="#infoaboutcallouts">next section.</a> The first argument for <b>pcre2_pattern_info()</b> is a pointer to the compiled pattern. The second argument specifies which piece of information is required, and the third argument is a pointer to a variable to receive the data. If the @@ -2043,7 +2148,7 @@ the following negative numbers: PCRE2_ERROR_BADOPTION the value of <i>what</i> was invalid PCRE2_ERROR_UNSET the requested field is not set </pre> -The "magic number" is placed at the start of each compiled pattern as an simple +The "magic number" is placed at the start of each compiled pattern as a simple check against passing an arbitrary memory pointer. Here is a typical call of <b>pcre2_pattern_info()</b>, to obtain the length of the compiled pattern: <pre> @@ -2100,7 +2205,7 @@ options returned for PCRE2_INFO_ALLOPTIONS. PCRE2_INFO_BACKREFMAX </pre> Return the number of the highest backreference in the pattern. The third -argument should point to an <b>uint32_t</b> variable. Named capture groups +argument should point to a <b>uint32_t</b> variable. Named capture groups acquire numbers as well as names, and these count towards the highest backreference. Backreferences such as \4 or \g{12} match the captured characters of the given group, but in addition, the check that a capture @@ -2118,7 +2223,7 @@ that \R matches only CR, LF, or CRLF. </pre> Return the highest capture group number in the pattern. In patterns where (?| is not used, this is also the total number of capture groups. The third -argument should point to an <b>uint32_t</b> variable. +argument should point to a <b>uint32_t</b> variable. <pre> PCRE2_INFO_DEPTHLIMIT </pre> @@ -2143,7 +2248,7 @@ returned. Otherwise NULL is returned. The third argument should point to a PCRE2_INFO_FIRSTCODETYPE </pre> Return information about the first code unit of any matched string, for a -non-anchored pattern. The third argument should point to an <b>uint32_t</b> +non-anchored pattern. The third argument should point to a <b>uint32_t</b> variable. If there is a fixed first value, for example, the letter "c" from a pattern such as (cat|cow|coyote), 1 is returned, and the value can be retrieved using PCRE2_INFO_FIRSTCODEUNIT. If there is no fixed first value, but it is @@ -2155,7 +2260,7 @@ is returned. </pre> Return the value of the first code unit of any matched string for a pattern where PCRE2_INFO_FIRSTCODETYPE returns 1; otherwise return 0. The third -argument should point to an <b>uint32_t</b> variable. In the 8-bit library, the +argument should point to a <b>uint32_t</b> variable. In the 8-bit library, the value is always less than 256. In the 16-bit library the value can be up to 0xffff. In the 32-bit library in UTF-32 mode the value can be up to 0x10ffff, and up to 0xffffffff when not using UTF-32 mode. @@ -2171,12 +2276,12 @@ pattern. Each additional capture group adds two PCRE2_SIZE variables. PCRE2_INFO_HASBACKSLASHC </pre> Return 1 if the pattern contains any instances of \C, otherwise 0. The third -argument should point to an <b>uint32_t</b> variable. +argument should point to a <b>uint32_t</b> variable. <pre> PCRE2_INFO_HASCRORLF </pre> Return 1 if the pattern contains any explicit matches for CR or LF characters, -otherwise 0. The third argument should point to an <b>uint32_t</b> variable. An +otherwise 0. The third argument should point to a <b>uint32_t</b> variable. An explicit match is either a literal CR or LF character, or \r or \n or one of the equivalent hexadecimal or octal escape sequences. <pre> @@ -2192,7 +2297,7 @@ defaulted by the caller of the match function. PCRE2_INFO_JCHANGED </pre> Return 1 if the (?J) or (?-J) option setting is used in the pattern, otherwise -0. The third argument should point to an <b>uint32_t</b> variable. (?J) and +0. The third argument should point to a <b>uint32_t</b> variable. (?J) and (?-J) set and unset the local PCRE2_DUPNAMES option, respectively. <pre> PCRE2_INFO_JITSIZE @@ -2204,7 +2309,7 @@ return zero. The third argument should point to a <b>size_t</b> variable. PCRE2_INFO_LASTCODETYPE </pre> Returns 1 if there is a rightmost literal code unit that must exist in any -matched string, other than at its start. The third argument should point to an +matched string, other than at its start. The third argument should point to a <b>uint32_t</b> variable. If there is no such value, 0 is returned. When 1 is returned, the code unit value itself can be retrieved using PCRE2_INFO_LASTCODEUNIT. For anchored patterns, a last literal value is @@ -2217,12 +2322,12 @@ PCRE2_INFO_LASTCODEUNIT), but for /^a\dz\d/ the returned value is 0. Return the value of the rightmost literal code unit that must exist in any matched string, other than at its start, for a pattern where PCRE2_INFO_LASTCODETYPE returns 1. Otherwise, return 0. The third argument -should point to an <b>uint32_t</b> variable. +should point to a <b>uint32_t</b> variable. <pre> PCRE2_INFO_MATCHEMPTY </pre> Return 1 if the pattern might match an empty string, otherwise 0. The third -argument should point to an <b>uint32_t</b> variable. When a pattern contains +argument should point to a <b>uint32_t</b> variable. When a pattern contains recursive subroutine calls it is not always possible to determine whether or not it can match an empty string. PCRE2 takes a cautious approach and returns 1 in such cases. @@ -2238,26 +2343,34 @@ defaulted by the caller of the match function. <pre> PCRE2_INFO_MAXLOOKBEHIND </pre> -Return the number of characters (not code units) in the longest lookbehind -assertion in the pattern. The third argument should point to a uint32_t -integer. This information is useful when doing multi-segment matching using the -partial matching facilities. Note that the simple assertions \b and \B -require a one-character lookbehind. \A also registers a one-character -lookbehind, though it does not actually inspect the previous character. This is -to ensure that at least one character from the old segment is retained when a -new segment is processed. Otherwise, if there are no lookbehinds in the -pattern, \A might match incorrectly at the start of a second or subsequent -segment. +A lookbehind assertion moves back a certain number of characters (not code +units) when it starts to process each of its branches. This request returns the +largest of these backward moves. The third argument should point to a uint32_t +integer. The simple assertions \b and \B require a one-character lookbehind +and cause PCRE2_INFO_MAXLOOKBEHIND to return 1 in the absence of anything +longer. \A also registers a one-character lookbehind, though it does not +actually inspect the previous character. +</P> +<P> +Note that this information is useful for multi-segment matching only +if the pattern contains no nested lookbehinds. For example, the pattern +(?<=a(?<=ba)c) returns a maximum lookbehind of 2, but when it is processed, the +first lookbehind moves back by two characters, matches one character, then the +nested lookbehind also moves back by two characters. This puts the matching +point three characters earlier than it was at the start. +PCRE2_INFO_MAXLOOKBEHIND is really only useful as a debugging tool. See the +<a href="pcre2partial.html"><b>pcre2partial</b></a> +documentation for a discussion of multi-segment matching. <pre> PCRE2_INFO_MINLENGTH </pre> If a minimum length for matching subject strings was computed, its value is -returned. Otherwise the returned value is 0. The value is a number of -characters, which in UTF mode may be different from the number of code units. -The third argument should point to an <b>uint32_t</b> variable. The value is a -lower bound to the length of any matching string. There may not be any strings -of that length that do actually match, but every string that does match is at -least that long. +returned. Otherwise the returned value is 0. This value is not computed when +PCRE2_NO_START_OPTIMIZE is set. The value is a number of characters, which in +UTF mode may be different from the number of code units. The third argument +should point to a <b>uint32_t</b> variable. The value is a lower bound to the +length of any matching string. There may not be any strings of that length that +do actually match, but every string that does match is at least that long. <pre> PCRE2_INFO_NAMECOUNT PCRE2_INFO_NAMEENTRYSIZE @@ -2653,15 +2766,22 @@ of JIT; it forces matching to be done by the interpreter. PCRE2_NO_UTF_CHECK </pre> When PCRE2_UTF is set at compile time, the validity of the subject as a UTF -string is checked by default when <b>pcre2_match()</b> is subsequently called. -If a non-zero starting offset is given, the check is applied only to that part -of the subject that could be inspected during matching, and there is a check -that the starting offset points to the first code unit of a character or to the -end of the subject. If there are no lookbehind assertions in the pattern, the -check starts at the starting offset. Otherwise, it starts at the length of the -longest lookbehind before the starting offset, or at the start of the subject -if there are not that many characters before the starting offset. Note that the -sequences \b and \B are one-character lookbehinds. +string is checked unless PCRE2_NO_UTF_CHECK is passed to <b>pcre2_match()</b> or +PCRE2_MATCH_INVALID_UTF was passed to <b>pcre2_compile()</b>. The latter special +case is discussed in detail in the +<a href="pcre2unicode.html"><b>pcre2unicode</b></a> +documentation. +</P> +<P> +In the default case, if a non-zero starting offset is given, the check is +applied only to that part of the subject that could be inspected during +matching, and there is a check that the starting offset points to the first +code unit of a character or to the end of the subject. If there are no +lookbehind assertions in the pattern, the check starts at the starting offset. +Otherwise, it starts at the length of the longest lookbehind before the +starting offset, or at the start of the subject if there are not that many +characters before the starting offset. Note that the sequences \b and \B are +one-character lookbehinds. </P> <P> The check is carried out before any other processing takes place, and a @@ -2674,31 +2794,37 @@ and <a href="pcre2unicode.html#utf32strings">UTF-32 strings</a> in the <a href="pcre2unicode.html"><b>pcre2unicode</b></a> -page. +documentation. </P> <P> -If you know that your subject is valid, and you want to skip these checks for +If you know that your subject is valid, and you want to skip this check for performance reasons, you can set the PCRE2_NO_UTF_CHECK option when calling <b>pcre2_match()</b>. You might want to do this for the second and subsequent -calls to <b>pcre2_match()</b> if you are making repeated calls to find other +calls to <b>pcre2_match()</b> if you are making repeated calls to find multiple matches in the same subject string. </P> <P> -<b>Warning:</b> When PCRE2_NO_UTF_CHECK is set, the effect of passing an invalid +<b>Warning:</b> Unless PCRE2_MATCH_INVALID_UTF was set at compile time, when +PCRE2_NO_UTF_CHECK is set at match time the effect of passing an invalid string as a subject, or an invalid value of <i>startoffset</i>, is undefined. -Your program may crash or loop indefinitely. +Your program may crash or loop indefinitely or give wrong results. <pre> PCRE2_PARTIAL_HARD PCRE2_PARTIAL_SOFT </pre> These options turn on the partial matching feature. A partial match occurs if the end of the subject string is reached successfully, but there are not enough -subject characters to complete the match. If this happens when -PCRE2_PARTIAL_SOFT (but not PCRE2_PARTIAL_HARD) is set, matching continues by -testing any remaining alternatives. Only if no complete match can be found is -PCRE2_ERROR_PARTIAL returned instead of PCRE2_ERROR_NOMATCH. In other words, -PCRE2_PARTIAL_SOFT specifies that the caller is prepared to handle a partial -match, but only if no complete match can be found. +subject characters to complete the match. In addition, either at least one +character must have been inspected or the pattern must contain a lookbehind, or +the pattern must be one that could match an empty string. +</P> +<P> +If this situation arises when PCRE2_PARTIAL_SOFT (but not PCRE2_PARTIAL_HARD) +is set, matching continues by testing any remaining alternatives. Only if no +complete match can be found is PCRE2_ERROR_PARTIAL returned instead of +PCRE2_ERROR_NOMATCH. In other words, PCRE2_PARTIAL_SOFT specifies that the +caller is prepared to handle a partial match, but only if no complete match can +be found. </P> <P> If PCRE2_PARTIAL_HARD is set, it overrides PCRE2_PARTIAL_SOFT. In this case, if @@ -3247,12 +3373,20 @@ same number causes an error at compile time. <b> PCRE2_SIZE *<i>outlengthptr</i>);</b> </P> <P> -This function calls <b>pcre2_match()</b> and then makes a copy of the subject -string in <i>outputbuffer</i>, replacing one or more parts that were matched -with the <i>replacement</i> string, whose length is supplied in <b>rlength</b>. -This can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. -The default is to perform just one replacement, but there is an option that -requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL below for details). +This function optionally calls <b>pcre2_match()</b> and then makes a copy of the +subject string in <i>outputbuffer</i>, replacing parts that were matched with +the <i>replacement</i> string, whose length is supplied in <b>rlength</b>. This +can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. There is an +option (see PCRE2_SUBSTITUTE_REPLACEMENT_ONLY below) to return just the +replacement string(s). The default action is to perform just one replacement if +the pattern matches, but there is an option that requests multiple replacements +(see PCRE2_SUBSTITUTE_GLOBAL below). +</P> +<P> +If successful, <b>pcre2_substitute()</b> returns the number of substitutions +that were carried out. This may be zero if no match was found, and is never +greater than one unless PCRE2_SUBSTITUTE_GLOBAL is set. A negative value is +returned if an error is detected. </P> <P> Matches in which a \K item in a lookahead in the pattern causes the match to @@ -3270,35 +3404,86 @@ functions from the match context, if provided, or else those that were used to allocate memory for the compiled code. </P> <P> -If an external <i>match_data</i> block is provided, its contents afterwards -are those set by the final call to <b>pcre2_match()</b>. For global changes, -this will have ended in a matching error. The contents of the ovector within -the match data block may or may not have been changed. +If <i>match_data</i> is not NULL and PCRE2_SUBSTITUTE_MATCHED is not set, the +provided block is used for all calls to <b>pcre2_match()</b>, and its contents +afterwards are the result of the final call. For global changes, this will +always be a no-match error. The contents of the ovector within the match data +block may or may not have been changed. +</P> +<P> +As well as the usual options for <b>pcre2_match()</b>, a number of additional +options can be set in the <i>options</i> argument of <b>pcre2_substitute()</b>. +One such option is PCRE2_SUBSTITUTE_MATCHED. When this is set, an external +<i>match_data</i> block must be provided, and it must have been used for an +external call to <b>pcre2_match()</b>. The data in the <i>match_data</i> block +(return code, offset vector) is used for the first substitution instead of +calling <b>pcre2_match()</b> from within <b>pcre2_substitute()</b>. This allows +an application to check for a match before choosing to substitute, without +having to repeat the match. </P> <P> -The <i>outlengthptr</i> argument must point to a variable that contains the -length, in code units, of the output buffer. If the function is successful, the -value is updated to contain the length of the new string, excluding the -trailing zero that is automatically added. +The contents of the externally supplied match data block are not changed when +PCRE2_SUBSTITUTE_MATCHED is set. If PCRE2_SUBSTITUTE_GLOBAL is also set, +<b>pcre2_match()</b> is called after the first substitution to check for further +matches, but this is done using an internally obtained match data block, thus +always leaving the external block unchanged. +</P> +<P> +The <i>code</i> argument is not used for matching before the first substitution +when PCRE2_SUBSTITUTE_MATCHED is set, but it must be provided, even when +PCRE2_SUBSTITUTE_GLOBAL is not set, because it contains information such as the +UTF setting and the number of capturing parentheses in the pattern. +</P> +<P> +The default action of <b>pcre2_substitute()</b> is to return a copy of the +subject string with matched substrings replaced. However, if +PCRE2_SUBSTITUTE_REPLACEMENT_ONLY is set, only the replacement substrings are +returned. In the global case, multiple replacements are concatenated in the +output buffer. Substitution callouts (see +<a href="#subcallouts">below)</a> +can be used to separate them if necessary. +</P> +<P> +The <i>outlengthptr</i> argument of <b>pcre2_substitute()</b> must point to a +variable that contains the length, in code units, of the output buffer. If the +function is successful, the value is updated to contain the length in code +units of the new string, excluding the trailing zero that is automatically +added. </P> <P> If the function is not successful, the value set via <i>outlengthptr</i> depends on the type of error. For syntax errors in the replacement string, the value is the offset in the replacement string where the error was detected. For other errors, the value is PCRE2_UNSET by default. This includes the case of the -output buffer being too small, unless PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set -(see below), in which case the value is the minimum length needed, including -space for the trailing zero. Note that in order to compute the required length, -<b>pcre2_substitute()</b> has to simulate all the matching and copying, instead -of giving an error return as soon as the buffer overflows. Note also that the -length is in code units, not bytes. +output buffer being too small, unless PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set. +</P> +<P> +PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output buffer is +too small. The default action is to return PCRE2_ERROR_NOMEMORY immediately. If +this option is set, however, <b>pcre2_substitute()</b> continues to go through +the motions of matching and substituting (without, of course, writing anything) +in order to compute the size of buffer that is needed. This value is passed +back via the <i>outlengthptr</i> variable, with the result of the function still +being PCRE2_ERROR_NOMEMORY. </P> <P> -In the replacement string, which is interpreted as a UTF string in UTF mode, -and is checked for UTF validity unless the PCRE2_NO_UTF_CHECK option is set, a -dollar character is an escape character that can specify the insertion of -characters from capture groups or names from (*MARK) or other control verbs -in the pattern. The following forms are always recognized: +Passing a buffer size of zero is a permitted way of finding out how much memory +is needed for given substitution. However, this does mean that the entire +operation is carried out twice. Depending on the application, it may be more +efficient to allocate a large buffer and free the excess afterwards, instead of +using PCRE2_SUBSTITUTE_OVERFLOW_LENGTH. +</P> +<P> +The replacement string, which is interpreted as a UTF string in UTF mode, is +checked for UTF validity unless PCRE2_NO_UTF_CHECK is set. An invalid UTF +replacement string causes an immediate return with the relevant UTF error code. +</P> +<P> +If PCRE2_SUBSTITUTE_LITERAL is set, the replacement string is not interpreted +in any way. By default, however, a dollar character is an escape character that +can specify the insertion of characters from capture groups and names from +(*MARK) or other control verbs in the pattern. The following forms are always +recognized: <pre> $$ insert a dollar character $<n> or ${<n>} insert the contents of group <n> @@ -3322,10 +3507,6 @@ facility can be used to perform simple simultaneous substitutions, as this apple lemon 2: pear orange </pre> -As well as the usual options for <b>pcre2_match()</b>, a number of additional -options can be set in the <i>options</i> argument of <b>pcre2_substitute()</b>. -</P> -<P> PCRE2_SUBSTITUTE_GLOBAL causes the function to iterate over the subject string, replacing every matching substring. If this option is not set, only the first matching substring is replaced. The search for matches takes place in the @@ -3337,7 +3518,7 @@ set in the match context, searching stops when that limit is reached. <P> You can restrict the effect of a global substitution to a portion of the subject string by setting either or both of <i>startoffset</i> and an offset -limit. Here is a \fPpcre2test\fP example: +limit. Here is a <b>pcre2test</b> example: <pre> /B/g,replace=!,use_offset_limit ABC ABC ABC ABC\=offset=3,offset_limit=12 @@ -3350,22 +3531,6 @@ CRLF is a valid newline sequence and the next two characters are CR, LF. In this case, the offset is advanced by two characters. </P> <P> -PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output buffer is -too small. The default action is to return PCRE2_ERROR_NOMEMORY immediately. If -this option is set, however, <b>pcre2_substitute()</b> continues to go through -the motions of matching and substituting (without, of course, writing anything) -in order to compute the size of buffer that is needed. This value is passed -back via the <i>outlengthptr</i> variable, with the result of the function still -being PCRE2_ERROR_NOMEMORY. -</P> -<P> -Passing a buffer size of zero is a permitted way of finding out how much memory -is needed for given substitution. However, this does mean that the entire -operation is carried out twice. Depending on the application, it may be more -efficient to allocate a large buffer and free the excess afterwards, instead of -using PCRE2_SUBSTITUTE_OVERFLOW_LENGTH. -</P> -<P> PCRE2_SUBSTITUTE_UNKNOWN_UNSET causes references to capture groups that do not appear in the pattern to be treated as unset groups. This option should be used with care, because it means that a typo in a group name or number no @@ -3400,13 +3565,16 @@ terminating a \Q quoted sequence) reverts to no case forcing. The sequences \u and \l force the next character (if it is a letter) to upper or lower case, respectively, and then the state automatically reverts to no case forcing. Case forcing applies to all inserted characters, including those from -capture groups and letters within \Q...\E quoted sequences. +capture groups and letters within \Q...\E quoted sequences. If either +PCRE2_UTF or PCRE2_UCP was set when the pattern was compiled, Unicode +properties are used for case forcing characters whose code points are greater +than 127. </P> <P> Note that case forcing sequences such as \U...\E do not nest. For example, the result of processing "\Uaa\LBB\Ecc\E" is "AAbbcc"; the final \E has no effect. Note also that the PCRE2_ALT_BSUX and PCRE2_EXTRA_ALT_BSUX options do -not apply to not apply to replacement strings. +not apply to replacement strings. </P> <P> The second effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more @@ -3439,14 +3607,17 @@ substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause unknown groups in the extended syntax forms to be treated as unset. </P> <P> -If successful, <b>pcre2_substitute()</b> returns the number of successful -matches. This may be zero if no matches were found, and is never greater than 1 -unless PCRE2_SUBSTITUTE_GLOBAL is set. +If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_UNKNOWN_UNSET, +PCRE2_SUBSTITUTE_UNSET_EMPTY, and PCRE2_SUBSTITUTE_EXTENDED are irrelevant and +are ignored. </P> +<br><b> +Substitution errors +</b><br> <P> -In the event of an error, a negative error code is returned. Except for -PCRE2_ERROR_NOMATCH (which is never returned), errors from <b>pcre2_match()</b> -are passed straight back. +In the event of an error, <b>pcre2_substitute()</b> returns a negative error +code. Except for PCRE2_ERROR_NOMATCH (which is never returned), errors from +<b>pcre2_match()</b> are passed straight back. </P> <P> PCRE2_ERROR_NOSUBSTRING is returned for a non-existent substring insertion, @@ -3464,6 +3635,10 @@ needed is returned via <i>outlengthptr</i>. Note that this does not happen by default. </P> <P> +PCRE2_ERROR_NULL is returned if PCRE2_SUBSTITUTE_MATCHED is set but the +<i>match_data</i> argument is NULL. +</P> +<P> PCRE2_ERROR_BADREPLACEMENT is used for miscellaneous syntax errors in the replacement string, with more particular errors being PCRE2_ERROR_BADREPESCAPE (invalid escape sequence), PCRE2_ERROR_REPMISSINGBRACE (closing curly bracket @@ -3477,7 +3652,7 @@ As for all PCRE2 errors, a text message that describes the error can be obtained by calling the <b>pcre2_get_error_message()</b> function (see "Obtaining a textual error message" <a href="#geterrormessage">above).</a> -</P> +<a name="subcallouts"></a></P> <br><b> Substitution callouts </b><br> @@ -3772,6 +3947,12 @@ This return is given if <b>pcre2_dfa_match()</b> encounters a condition item that uses a backreference for the condition, or a test for recursion in a specific capture group. These are not supported. <pre> + PCRE2_ERROR_DFA_UINVALID_UTF +</pre> +This return is given if <b>pcre2_dfa_match()</b> is called for a pattern that +was compiled with PCRE2_MATCH_INVALID_UTF. This is not supported for DFA +matching. +<pre> PCRE2_ERROR_DFA_WSSIZE </pre> This return is given if <b>pcre2_dfa_match()</b> runs out of space in the @@ -3808,9 +3989,9 @@ Cambridge, England. </P> <br><a name="SEC42" href="#TOC1">REVISION</a><br> <P> -Last updated: 14 February 2019 +Last updated: 04 November 2020 <br> -Copyright © 1997-2019 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2build.html b/dist2/doc/html/pcre2build.html index 13d9da22..a206b232 100644 --- a/dist2/doc/html/pcre2build.html +++ b/dist2/doc/html/pcre2build.html @@ -128,7 +128,7 @@ To build it without Unicode support, add --disable-unicode </pre> to the <b>configure</b> command. This setting applies to all three libraries. It -is not possible to build one library with Unicode support, and another without, +is not possible to build one library with Unicode support and another without in the same configuration. </P> <P> @@ -188,11 +188,11 @@ which enables the use of an execmem allocator in JIT that is compatible with SELinux. This has no effect if JIT is not enabled. See the <a href="pcre2jit.html"><b>pcre2jit</b></a> documentation for a discussion of JIT usage. When JIT support is enabled, -pcre2grep automatically makes use of it, unless you add +<b>pcre2grep</b> automatically makes use of it, unless you add <pre> --disable-pcre2grep-jit </pre> -to the "configure" command. +to the <b>configure</b> command. </P> <br><a name="SEC8" href="#TOC1">NEWLINE RECOGNITION</a><br> <P> @@ -321,7 +321,7 @@ As well as applying to <b>pcre2_match()</b>, the depth limit also controls the depth of recursive function calls in <b>pcre2_dfa_match()</b>. These are used for lookaround assertions, atomic groups, and recursion within patterns. The limit does not apply to JIT matching. -</P> +<a name="createtables"></a></P> <br><a name="SEC12" href="#TOC1">CREATING CHARACTER TABLES AT BUILD TIME</a><br> <P> PCRE2 uses fixed tables for processing characters whose code points are less @@ -332,12 +332,34 @@ only. If you add --enable-rebuild-chartables </pre> to the <b>configure</b> command, the distributed tables are no longer used. -Instead, a program called <b>dftables</b> is compiled and run. This outputs the -source for new set of tables, created in the default locale of your C run-time -system. This method of replacing the tables does not work if you are cross -compiling, because <b>dftables</b> is run on the local host. If you need to -create alternative tables when cross compiling, you will have to do so "by -hand". +Instead, a program called <b>pcre2_dftables</b> is compiled and run. This +outputs the source for new set of tables, created in the default locale of your +C run-time system. This method of replacing the tables does not work if you are +cross compiling, because <b>pcre2_dftables</b> needs to be run on the local +host and therefore not compiled with the cross compiler. +</P> +<P> +If you need to create alternative tables when cross compiling, you will have to +do so "by hand". There may also be other reasons for creating tables manually. +To cause <b>pcre2_dftables</b> to be built on the local host, run a normal +compiling command, and then run the program with the output file as its +argument, for example: +<pre> + cc src/pcre2_dftables.c -o pcre2_dftables + ./pcre2_dftables src/pcre2_chartables.c +</pre> +This builds the tables in the default locale of the local host. If you want to +specify a locale, you must use the -L option: +<pre> + LC_ALL=fr_FR ./pcre2_dftables -L src/pcre2_chartables.c +</pre> +You can also specify -b (with or without -L). This causes the tables to be +written in binary instead of as source code. A set of binary tables can be +loaded into memory by an application and passed to <b>pcre2_compile()</b> in the +same way as tables created by calling <b>pcre2_maketables()</b>. The tables are +just a string of bytes, independent of hardware characteristics such as +endianness. This means they can be bundled with an application that runs in +different environments, to ensure consistent behaviour. </P> <br><a name="SEC13" href="#TOC1">USING EBCDIC CODE</a><br> <P> @@ -414,7 +436,7 @@ default parameter values by adding, for example, --with-pcre2grep-bufsize=51200 --with-pcre2grep-max-bufsize=2097152 </pre> -to the <b>configure</b> command. The caller of \fPpcre2grep\fP can override +to the <b>configure</b> command. The caller of <b>pcre2grep</b> can override these values by using --buffer-size and --max-buffer-size on the command line. </P> <br><a name="SEC17" href="#TOC1">PCRE2TEST OPTION FOR LIBREADLINE SUPPORT</a><br> @@ -538,7 +560,7 @@ support these modifiers. If <pre> --disable-percent-zt </pre> -is specified, no use is made of the z or t modifiers. Instead or %td or %zu, +is specified, no use is made of the z or t modifiers. Instead of %td or %zu, %lu is used, with a cast for size_t values. </P> <br><a name="SEC22" href="#TOC1">SUPPORT FOR FUZZERS</a><br> @@ -592,9 +614,9 @@ Cambridge, England. </P> <br><a name="SEC26" href="#TOC1">REVISION</a><br> <P> -Last updated: 03 March 2019 +Last updated: 20 March 2020 <br> -Copyright © 1997-2019 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2compat.html b/dist2/doc/html/pcre2compat.html index 7d728f58..54fb6437 100644 --- a/dist2/doc/html/pcre2compat.html +++ b/dist2/doc/html/pcre2compat.html @@ -16,10 +16,10 @@ please consult the man page, in case the conversion went wrong. DIFFERENCES BETWEEN PCRE2 AND PERL </b><br> <P> -This document describes the differences in the ways that PCRE2 and Perl handle -regular expressions. The differences described here are with respect to Perl -versions 5.26, but as both Perl and PCRE2 are continually changing, the -information may sometimes be out of date. +This document describes some of the differences in the ways that PCRE2 and Perl +handle regular expressions. The differences described here are with respect to +Perl version 5.32.0, but as both Perl and PCRE2 are continually changing, the +information may at times be out of date. </P> <P> 1. PCRE2 has only a subset of Perl's Unicode support. Details of what it does @@ -33,12 +33,15 @@ they do not mean what you might think. For example, (?!a){3} does not assert that the next three characters are not "a". It just asserts that the next character is not "a" three times (in principle; PCRE2 optimizes this to run the assertion just once). Perl allows some repeat quantifiers on other assertions, -for example, \b* (but not \b{3}), but these do not seem to have any use. +for example, \b* (but not \b{3}, though oddly it does allow ^{3}), but these +do not seem to have any use. PCRE2 does not allow any kind of quantifier on +non-lookaround assertions. </P> <P> 3. Capture groups that occur inside negative lookaround assertions are counted, but their entries in the offsets vector are set only when a negative assertion is a condition that has a matching branch (that is, the condition is false). +Perl may set such capture groups in other circumstances. </P> <P> 4. The following Perl escape sequences are not supported: \F, \l, \L, \u, @@ -56,10 +59,12 @@ interprets them. built with Unicode support (the default). The properties that can be tested with \p and \P are limited to the general category properties such as Lu and Nd, script names such as Greek or Han, and the derived properties Any and L&. -PCRE2 does support the Cs (surrogate) property, which Perl does not; the Perl -documentation says "Because Perl hides the need for the user to understand the -internal representation of Unicode characters, there is no need to implement -the somewhat messy concept of surrogates." +Both PCRE2 and Perl support the Cs (surrogate) property, but in PCRE2 its use +is limited. See the +<a href="pcre2pattern.html"><b>pcre2pattern</b></a> +documentation for details. The long synonyms for property names that Perl +supports (such as \p{Letter}) are not supported by PCRE2, nor is it permitted +to prefix any of these properties with "Is". </P> <P> 6. PCRE2 supports the \Q...\E escape for quoting substrings. Characters @@ -79,7 +84,8 @@ other character. Note the following examples: \QA\B\E A\B A\B \Q\\E \ \\E </pre> -The \Q...\E sequence is recognized both inside and outside character classes. +The \Q...\E sequence is recognized both inside and outside character classes +by both PCRE2 and Perl. </P> <P> 7. Fairly obviously, PCRE2 does not support the (?{code}) and (??{code}) @@ -94,13 +100,13 @@ to PCRE2 release 10.23, but from release 10.30 this changed, and backtracking into subroutine calls is now supported, as in Perl. </P> <P> -9. If any of the backtracking control verbs are used in a group that is called -as a subroutine (whether or not recursively), their effect is confined to that -group; it does not extend to the surrounding pattern. This is not always the -case in Perl. In particular, if (*THEN) is present in a group that is called as -a subroutine, its action is limited to that group, even if the group does not -contain any | characters. Note that such groups are processed as anchored -at the point where they are tested. +9. In PCRE2, if any of the backtracking control verbs are used in a group that +is called as a subroutine (whether or not recursively), their effect is +confined to that group; it does not extend to the surrounding pattern. This is +not always the case in Perl. In particular, if (*THEN) is present in a group +that is called as a subroutine, its action is limited to that group, even if +the group does not contain any | characters. Note that such groups are +processed as anchored at the point where they are tested. </P> <P> 10. If a pattern contains more than one backtracking control verb, the first @@ -110,55 +116,56 @@ triggers (*PRUNE). Perl's behaviour is more complex; in many cases it is the same as PCRE2, but there are cases where it differs. </P> <P> -11. Most backtracking verbs in assertions have their normal actions. They are -not confined to the assertion. -</P> -<P> -12. There are some differences that are concerned with the settings of captured +11. There are some differences that are concerned with the settings of captured strings when part of a pattern is repeated. For example, matching "aba" against the pattern /^(a(b)?)+$/ in Perl leaves $2 unset, but in PCRE2 it is set to "b". </P> <P> -13. PCRE2's handling of duplicate capture group numbers and names is not as +12. PCRE2's handling of duplicate capture group numbers and names is not as general as Perl's. This is a consequence of the fact the PCRE2 works internally just with numbers, using an external table to translate between numbers and -names. In particular, a pattern such as (?|(?<a>A)|(?<b>B), where the two +names. In particular, a pattern such as (?|(?<a>A)|(?<b>B)), where the two capture groups have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which group matched, because both names map to capture group number 1. To avoid this confusing situation, an error is given at compile time. </P> <P> -14. Perl used to recognize comments in some places that PCRE2 does not, for +13. Perl used to recognize comments in some places that PCRE2 does not, for example, between the ( and ? at the start of a group. If the /x modifier is set, Perl allowed white space between ( and ? though the latest Perls give an error (for a while it was just deprecated). There may still be some cases where Perl behaves differently. </P> <P> -15. Perl, when in warning mode, gives warnings for character classes such as +14. Perl, when in warning mode, gives warnings for character classes such as [A-\d] or [a-[:digit:]]. It then treats the hyphens as literals. PCRE2 has no warning features, so it gives an error in these cases because they are almost certainly user mistakes. </P> <P> -16. In PCRE2, the upper/lower case character properties Lu and Ll are not +15. In PCRE2, the upper/lower case character properties Lu and Ll are not affected when case-independent matching is specified. For example, \p{Lu} always matches an upper case letter. I think Perl has changed in this respect; -in the release at the time of writing (5.24), \p{Lu} and \p{Ll} match all +in the release at the time of writing (5.32), \p{Lu} and \p{Ll} match all letters, regardless of case, when case independence is specified. </P> <P> +16. From release 5.32.0, Perl locks out the use of \K in lookaround +assertions. In PCRE2, \K is acted on when it occurs in positive assertions, +but is ignored in negative assertions. +</P> +<P> 17. PCRE2 provides some extensions to the Perl regular expression facilities. -Perl 5.10 includes new features that are not in earlier versions of Perl, some +Perl 5.10 included new features that were not in earlier versions of Perl, some of which (such as named parentheses) were in PCRE2 for some time before. This -list is with respect to Perl 5.26: +list is with respect to Perl 5.32: <br> <br> (a) Although lookbehind assertions in PCRE2 must match fixed length strings, -each alternative branch of a lookbehind assertion can match a different length -of string. Perl requires them all to have the same length. +each alternative toplevel branch of a lookbehind assertion can match a +different length of string. Perl requires them all to have the same length. <br> <br> (b) From PCRE2 10.23, backreferences to groups of fixed length are supported @@ -203,8 +210,13 @@ different way and is not Perl-compatible. <br> <br> (l) PCRE2 recognizes some special sequences such as (*CR) or (*NO_JIT) at -the start of a pattern that set overall options that cannot be changed within +the start of a pattern. These set overall options that cannot be changed within the pattern. +<br> +<br> +(m) PCRE2 supports non-atomic positive lookaround assertions. This is an +extension to the lookaround facilities. The default, Perl-compatible +lookarounds are atomic. </P> <P> 18. The Perl /a modifier restricts /d numbers to pure ascii, and the /aa @@ -234,7 +246,7 @@ Cambridge, England. REVISION </b><br> <P> -Last updated: 12 February 2019 +Last updated: 06 October 2020 <br> Copyright © 1997-2019 University of Cambridge. <br> diff --git a/dist2/doc/html/pcre2demo.html b/dist2/doc/html/pcre2demo.html index 72754d3e..08b21901 100644 --- a/dist2/doc/html/pcre2demo.html +++ b/dist2/doc/html/pcre2demo.html @@ -104,12 +104,11 @@ uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE subject_length; -size_t subject_length; pcre2_match_data *match_data; - /************************************************************************** * First, sort out the command line. There is only one possible option at * * the moment, "-g" to request repeated matching to find all occurrences, * @@ -138,12 +137,14 @@ if (argc - i != 2) return 1; } -/* As pattern and subject are char arguments, they can be straightforwardly -cast to PCRE2_SPTR as we are working in 8-bit code units. */ +/* Pattern and subject are char arguments, so they can be straightforwardly +cast to PCRE2_SPTR because we are working in 8-bit code units. The subject +length is cast to PCRE2_SIZE for completeness, though PCRE2_SIZE is in fact +defined to be size_t. */ pattern = (PCRE2_SPTR)argv[i]; subject = (PCRE2_SPTR)argv[i+1]; -subject_length = strlen((char *)subject); +subject_length = (PCRE2_SIZE)strlen((char *)subject); /************************************************************************* @@ -172,17 +173,22 @@ if (re == NULL) /************************************************************************* -* If the compilation succeeded, we call PCRE again, in order to do a * +* If the compilation succeeded, we call PCRE2 again, in order to do a * * pattern match against the subject string. This does just ONE match. If * * further matching is needed, it will be done below. Before running the * -* match we must set up a match_data block for holding the result. * +* match we must set up a match_data block for holding the result. Using * +* pcre2_match_data_create_from_pattern() ensures that the block is * +* exactly the right size for the number of capturing parentheses in the * +* pattern. If you need to know the actual size of a match_data block as * +* a number of bytes, you can find it like this: * +* * +* PCRE2_SIZE match_data_size = pcre2_get_match_data_size(match_data); * *************************************************************************/ -/* Using this function ensures that the block is exactly the right size for -the number of capturing parentheses in the pattern. */ - match_data = pcre2_match_data_create_from_pattern(re, NULL); +/* Now run the match. */ + rc = pcre2_match( re, /* the compiled pattern */ subject, /* the subject string */ @@ -205,7 +211,7 @@ if (rc < 0) default: printf("Matching error %d\n", rc); break; } pcre2_match_data_free(match_data); /* Release memory used for the match */ - pcre2_code_free(re); /* data and the compiled pattern. */ + pcre2_code_free(re); /* data and the compiled pattern. */ return 1; } @@ -249,7 +255,7 @@ application you might want to do things other than print them. */ for (i = 0; i < rc; i++) { PCRE2_SPTR substring_start = subject + ovector[2*i]; - size_t substring_length = ovector[2*i+1] - ovector[2*i]; + PCRE2_SIZE substring_length = ovector[2*i+1] - ovector[2*i]; printf("%2d: %.*s\n", i, (int)substring_length, (char *)substring_start); } diff --git a/dist2/doc/html/pcre2grep.html b/dist2/doc/html/pcre2grep.html index d66cee34..995e0eab 100644 --- a/dist2/doc/html/pcre2grep.html +++ b/dist2/doc/html/pcre2grep.html @@ -111,8 +111,8 @@ matching substrings, or if <b>--only-matching</b>, <b>--file-offsets</b>, or (either shown literally, or as an offset), scanning resumes immediately following the match, so that further matches on the same line can be found. If there are multiple patterns, they are all tried on the remainder of the line, -but patterns that follow the one that matched are not tried on the earlier part -of the line. +but patterns that follow the one that matched are not tried on the earlier +matched part of the line. </P> <P> This behaviour means that the order in which multiple patterns are specified @@ -146,11 +146,10 @@ ignored. <br><a name="SEC4" href="#TOC1">BINARY FILES</a><br> <P> By default, a file that contains a binary zero byte within the first 1024 bytes -is identified as a binary file, and is processed specially. (GNU grep -identifies binary files in this manner.) However, if the newline type is -specified as "nul", that is, the line terminator is a binary zero, the test for -a binary file is not applied. See the <b>--binary-files</b> option for a means -of changing the way binary files are handled. +is identified as a binary file, and is processed specially. However, if the +newline type is specified as NUL, that is, the line terminator is a binary +zero, the test for a binary file is not applied. See the <b>--binary-files</b> +option for a means of changing the way binary files are handled. </P> <br><a name="SEC5" href="#TOC1">BINARY ZEROS IN PATTERNS</a><br> <P> @@ -443,8 +442,8 @@ Ignore upper/lower case distinctions during comparisons. <P> <b>--include</b>=<i>pattern</i> If any <b>--include</b> patterns are specified, the only files that are -processed are those that match one of the patterns (and do not match an -<b>--exclude</b> pattern). This option does not affect directories, but it +processed are those whose names match one of the patterns and do not match an +<b>--exclude</b> pattern. This option does not affect directories, but it applies to all files, whether listed on the command line, obtained from <b>--file-list</b>, or by scanning a directory. The pattern is a PCRE2 regular expression, and is matched against the final component of the file name, not @@ -463,8 +462,8 @@ may be given any number of times; all the files are read. <P> <b>--include-dir</b>=<i>pattern</i> If any <b>--include-dir</b> patterns are specified, the only directories that -are processed are those that match one of the patterns (and do not match an -<b>--exclude-dir</b> pattern). This applies to all directories, whether listed +are processed are those whose names match one of the patterns and do not match +an <b>--exclude-dir</b> pattern. This applies to all directories, whether listed on the command line, obtained from <b>--file-list</b>, or by scanning a parent directory. The pattern is a PCRE2 regular expression, and is matched against the final component of the directory name, not the entire path. The <b>-F</b>, @@ -487,8 +486,9 @@ a separate line. Searching normally stops as soon as a matching line is found in a file. However, if the <b>-c</b> (count) option is also used, matching continues in order to obtain the correct count, and those files that have at least one match are listed along with their counts. Using this option with -<b>-c</b> is a way of suppressing the listing of files with no matches. This -opeion overrides any previous <b>-H</b>, <b>-h</b>, or <b>-L</b> options. +<b>-c</b> is a way of suppressing the listing of files with no matches that +occurs with <b>-c</b> on its own. This option overrides any previous <b>-H</b>, +<b>-h</b>, or <b>-L</b> options. </P> <P> <b>--label</b>=<i>name</i> @@ -501,8 +501,8 @@ short form for this option. When this option is given, non-compressed input is read and processed line by line, and the output is flushed after each write. By default, input is read in large chunks, unless <b>pcre2grep</b> can determine that it is reading from a -terminal (which is currently possible only in Unix-like environments or -Windows). Output to terminal is normally automatically flushed by the operating +terminal, which is currently possible only in Unix-like environments or +Windows. Output to terminal is normally automatically flushed by the operating system. This option can be useful when the input or output is attached to a pipe and you do not want <b>pcre2grep</b> to buffer up large amounts of data. However, its use will affect performance, and the <b>-M</b> (multiline) option @@ -528,6 +528,49 @@ locale is specified, the PCRE2 library's default (usually the "C" locale) is used. There is no short form for this option. </P> <P> +<b>-M</b>, <b>--multiline</b> +Allow patterns to match more than one line. When this option is set, the PCRE2 +library is called in "multiline" mode. This allows a matched string to extend +past the end of a line and continue on one or more subsequent lines. Patterns +used with <b>-M</b> may usefully contain literal newline characters and internal +occurrences of ^ and $ characters. The output for a successful match may +consist of more than one line. The first line is the line in which the match +started, and the last line is the line in which the match ended. If the matched +string ends with a newline sequence, the output ends at the end of that line. +If <b>-v</b> is set, none of the lines in a multi-line match are output. Once a +match has been handled, scanning restarts at the beginning of the line after +the one in which the match ended. +<br> +<br> +The newline sequence that separates multiple lines must be matched as part of +the pattern. For example, to find the phrase "regular expression" in a file +where "regular" might be at the end of a line and "expression" at the start of +the next line, you could use this command: +<pre> + pcre2grep -M 'regular\s+expression' <file> +</pre> +The \s escape sequence matches any white space character, including newlines, +and is followed by + so as to match trailing white space on the first line as +well as possibly handling a two-character newline sequence. +<br> +<br> +There is a limit to the number of lines that can be matched, imposed by the way +that <b>pcre2grep</b> buffers the input file as it scans it. With a sufficiently +large processing buffer, this should not be a problem, but the <b>-M</b> option +does not work when input is read line by line (see <b>--line-buffered</b>.) +</P> +<P> +<b>-m</b> <i>number</i>, <b>--max-count</b>=<i>number</i> +Stop processing after finding <i>number</i> matching lines, or non-matching +lines if <b>-v</b> is also set. Any trailing context lines are output after the +final match. In multiline mode, each multiline match counts as just one line +for this purpose. If this limit is reached when reading the standard input from +a regular file, the file is left positioned just after the last matching line. +If <b>-c</b> is also set, the count that is output is never greater than +<i>number</i>. This option has no effect if used with <b>-L</b>, <b>-l</b>, or +<b>-q</b>, or when just checking for a match in a binary file. +</P> +<P> <b>--match-limit</b>=<i>number</i> Processing some regular expression patterns may take a very long time to search for all possible matching strings. Others may require a very large amount of @@ -562,64 +605,39 @@ when the PCRE2 library is compiled; if they are not specified, the defaults are very large and so effectively unlimited. </P> <P> -\fB--max-buffer-size=<i>number</i> +<b>--max-buffer-size</b>=<i>number</i> This limits the expansion of the processing buffer, whose initial size can be set by <b>--buffer-size</b>. The maximum buffer size is silently forced to be no smaller than the starting buffer size. </P> <P> -<b>-M</b>, <b>--multiline</b> -Allow patterns to match more than one line. When this option is set, the PCRE2 -library is called in "multiline" mode. This allows a matched string to extend -past the end of a line and continue on one or more subsequent lines. Patterns -used with <b>-M</b> may usefully contain literal newline characters and internal -occurrences of ^ and $ characters. The output for a successful match may -consist of more than one line. The first line is the line in which the match -started, and the last line is the line in which the match ended. If the matched -string ends with a newline sequence, the output ends at the end of that line. -If <b>-v</b> is set, none of the lines in a multi-line match are output. Once a -match has been handled, scanning restarts at the beginning of the line after -the one in which the match ended. -<br> -<br> -The newline sequence that separates multiple lines must be matched as part of -the pattern. For example, to find the phrase "regular expression" in a file -where "regular" might be at the end of a line and "expression" at the start of -the next line, you could use this command: +<b>-N</b> <i>newline-type</i>, <b>--newline</b>=<i>newline-type</i> +Six different conventions for indicating the ends of lines in scanned files are +supported. For example: <pre> - pcre2grep -M 'regular\s+expression' <file> + pcre2grep -N CRLF 'some pattern' <file> </pre> -The \s escape sequence matches any white space character, including newlines, -and is followed by + so as to match trailing white space on the first line as -well as possibly handling a two-character newline sequence. -<br> -<br> -There is a limit to the number of lines that can be matched, imposed by the way -that <b>pcre2grep</b> buffers the input file as it scans it. With a sufficiently -large processing buffer, this should not be a problem, but the <b>-M</b> option -does not work when input is read line by line (see \fP--line-buffered\fP.) -</P> -<P> -<b>-N</b> <i>newline-type</i>, <b>--newline</b>=<i>newline-type</i> -The PCRE2 library supports five different conventions for indicating -the ends of lines. They are the single-character sequences CR (carriage return) -and LF (linefeed), the two-character sequence CRLF, an "anycrlf" convention, -which recognizes any of the preceding three types, and an "any" convention, in -which any Unicode line ending sequence is assumed to end a line. The Unicode -sequences are the three just mentioned, plus VT (vertical tab, U+000B), FF -(form feed, U+000C), NEL (next line, U+0085), LS (line separator, U+2028), and -PS (paragraph separator, U+2029). +The newline type may be specified in upper, lower, or mixed case. If the +newline type is NUL, lines are separated by binary zero characters. The other +types are the single-character sequences CR (carriage return) and LF +(linefeed), the two-character sequence CRLF, an "anycrlf" type, which +recognizes any of the preceding three types, and an "any" type, for which any +Unicode line ending sequence is assumed to end a line. The Unicode sequences +are the three just mentioned, plus VT (vertical tab, U+000B), FF (form feed, +U+000C), NEL (next line, U+0085), LS (line separator, U+2028), and PS +(paragraph separator, U+2029). <br> <br> When the PCRE2 library is built, a default line-ending sequence is specified. This is normally the standard sequence for the operating system. Unless otherwise specified by this option, <b>pcre2grep</b> uses the library's default. -The possible values for this option are CR, LF, CRLF, ANYCRLF, or ANY. This -makes it possible to use <b>pcre2grep</b> to scan files that have come from -other environments without having to modify their line endings. If the data -that is being scanned does not agree with the convention set by this option, -<b>pcre2grep</b> may behave in strange ways. Note that this option does not -apply to files specified by the <b>-f</b>, <b>--exclude-from</b>, or +<br> +<br> +This option makes it possible to use <b>pcre2grep</b> to scan files that have +come from other environments without having to modify their line endings. If +the data that is being scanned does not agree with the convention set by this +option, <b>pcre2grep</b> may behave in strange ways. Note that this option does +not apply to files specified by the <b>-f</b>, <b>--exclude-from</b>, or <b>--include-from</b> options, which are expected to use the operating system's standard newline sequence. </P> @@ -641,29 +659,41 @@ It should never be needed in normal use. </P> <P> <b>-O</b> <i>text</i>, <b>--output</b>=<i>text</i> -When there is a match, instead of outputting the whole line that matched, -output just the given text. This option is mutually exclusive with -<b>--only-matching</b>, <b>--file-offsets</b>, and <b>--line-offsets</b>. Escape -sequences starting with a dollar character may be used to insert the contents -of the matched part of the line and/or captured substrings into the text. +When there is a match, instead of outputting the line that matched, output just +the text specified in this option, followed by an operating-system standard +newline. In this mode, no context is shown. That is, the <b>-A</b>, <b>-B</b>, +and <b>-C</b> options are ignored. The <b>--newline</b> option has no effect on +this option, which is mutually exclusive with <b>--only-matching</b>, +<b>--file-offsets</b>, and <b>--line-offsets</b>. However, like +<b>--only-matching</b>, if there is more than one match in a line, each of them +causes a line of output. +<br> +<br> +Escape sequences starting with a dollar character may be used to insert the +contents of the matched part of the line and/or captured substrings into the +text. <br> <br> -$<digits> or ${<digits>} is replaced by the captured -substring of the given decimal number; zero substitutes the whole match. If -the number is greater than the number of capturing substrings, or if the -capture is unset, the replacement is empty. +$<digits> or ${<digits>} is replaced by the captured substring of the given +decimal number; zero substitutes the whole match. If the number is greater than +the number of capturing substrings, or if the capture is unset, the replacement +is empty. <br> <br> $a is replaced by bell; $b by backspace; $e by escape; $f by form feed; $n by newline; $r by carriage return; $t by tab; $v by vertical tab. <br> <br> -$o<digits> is replaced by the character represented by the given octal -number; up to three digits are processed. +$o<digits> or $o{<digits>} is replaced by the character whose code point is the +given octal number. In the first form, up to three octal digits are processed. +When more digits are needed in Unicode mode to specify a wide character, the +second form must be used. <br> <br> -$x<digits> is replaced by the character represented by the given hexadecimal -number; up to two digits are processed. +$x<digits> or $x{<digits>} is replaced by the character represented by the +given hexadecimal number. In the first form, up to two hexadecimal digits are +processed. When more digits are needed in Unicode mode to specify a wide +character, the second form must be used. <br> <br> Any other character is substituted by itself. In particular, $$ is replaced by @@ -685,20 +715,32 @@ otherwise empty line. This option is mutually exclusive with <b>--output</b>, <P> <b>-o</b><i>number</i>, <b>--only-matching</b>=<i>number</i> Show only the part of the line that matched the capturing parentheses of the -given number. Up to 32 capturing parentheses are supported, and -o0 is -equivalent to <b>-o</b> without a number. Because these options can be given -without an argument (see above), if an argument is present, it must be given in -the same shell item, for example, -o3 or --only-matching=2. The comments given -for the non-argument case above also apply to this option. If the specified -capturing parentheses do not exist in the pattern, or were not set in the -match, nothing is output unless the file name or line number are being output. +given number. Up to 50 capturing parentheses are supported by default. This +limit can be changed via the <b>--om-capture</b> option. A pattern may contain +any number of capturing parentheses, but only those whose number is within the +limit can be accessed by <b>-o</b>. An error occurs if the number specified by +<b>-o</b> is greater than the limit. +<br> +<br> +-o0 is the same as <b>-o</b> without a number. Because these options can be +given without an argument (see above), if an argument is present, it must be +given in the same shell item, for example, -o3 or --only-matching=2. The +comments given for the non-argument case above also apply to this option. If +the specified capturing parentheses do not exist in the pattern, or were not +set in the match, nothing is output unless the file name or line number are +being output. <br> <br> If this option is given multiple times, multiple substrings are output for each match, in the order the options are given, and all on one line. For example, -o3 -o1 -o3 causes the substrings matched by capturing parentheses 3 and 1 and then 3 again to be output. By default, there is no separator (but see the next -option). +but one option). +</P> +<P> +<b>--om-capture</b>=<i>number</i> +Set the number of capturing parentheses that can be accessed by <b>-o</b>. The +default is 50. </P> <P> <b>--om-separator</b>=<i>text</i> @@ -720,7 +762,8 @@ option to "recurse". </P> <P> <b>--recursion-limit</b>=<i>number</i> -See <b>--match-limit</b> above. +This is an obsolete synonym for <b>--depth-limit</b>. See <b>--match-limit</b> +above for details. </P> <P> <b>-s</b>, <b>--no-messages</b> @@ -741,11 +784,23 @@ ignored when used with <b>-L</b> (list files without matches), because the grand total would always be zero. </P> <P> -<b>-u</b>, <b>--utf-8</b> +<b>-u</b>, <b>--utf</b> Operate in UTF-8 mode. This option is available only if PCRE2 has been compiled with UTF-8 support. All patterns (including those for any <b>--exclude</b> and -<b>--include</b> options) and all subject lines that are scanned must be valid -strings of UTF-8 characters. +<b>--include</b> options) and all lines that are scanned must be valid strings +of UTF-8 characters. If an invalid UTF-8 string is encountered, an error +occurs. +</P> +<P> +<b>-U</b>, <b>--utf-allow-invalid</b> +As <b>--utf</b>, but in addition subject lines may contain invalid UTF-8 code +unit sequences. These can never form part of any pattern match. Patterns +themselves, however, must still be valid UTF-8 strings. This facility allows +valid UTF-8 strings to be sought within arbitrary byte sequences in executable +or other binary files. For more details about matching in non-valid UTF-8 +strings, see the +<a href="pcre2unicode.html"><b>pcre2unicode</b>(3)</a> +documentation. </P> <P> <b>-V</b>, <b>--version</b> @@ -756,7 +811,9 @@ ignored. <P> <b>-v</b>, <b>--invert-match</b> Invert the sense of the match, so that lines which do <i>not</i> match any of -the patterns are the ones that are found. +the patterns are the ones that are found. When this option is set, options such +as <b>--only-matching</b> and <b>--output</b>, which specify parts of a match +that are to be output, are ignored. </P> <P> <b>-w</b>, <b>--word-regex</b>, <b>--word-regexp</b> @@ -786,16 +843,27 @@ by the <b>--locale</b> option. If no locale is set, the PCRE2 library's default <br><a name="SEC8" href="#TOC1">NEWLINES</a><br> <P> The <b>-N</b> (<b>--newline</b>) option allows <b>pcre2grep</b> to scan files with -different newline conventions from the default. Any parts of the input files -that are written to the standard output are copied identically, with whatever -newline sequences they have in the input. However, the setting of this option -affects only the way scanned files are processed. It does not affect the -interpretation of files specified by the <b>-f</b>, <b>--file-list</b>, -<b>--exclude-from</b>, or <b>--include-from</b> options, nor does it affect the -way in which <b>pcre2grep</b> writes informational messages to the standard -error and output streams. For these it uses the string "\n" to indicate -newlines, relying on the C I/O library to convert this to an appropriate -sequence. +newline conventions that differ from the default. This option affects only the +way scanned files are processed. It does not affect the interpretation of files +specified by the <b>-f</b>, <b>--file-list</b>, <b>--exclude-from</b>, or +<b>--include-from</b> options. +</P> +<P> +Any parts of the scanned input files that are written to the standard output +are copied with whatever newline sequences they have in the input. However, if +the final line of a file is output, and it does not end with a newline +sequence, a newline sequence is added. If the newline setting is CR, LF, CRLF +or NUL, that line ending is output; for the other settings (ANYCRLF or ANY) a +single NL is used. +</P> +<P> +The newline setting does not affect the way in which <b>pcre2grep</b> writes +newlines in informational messages to the standard output and error streams. +Under Windows, the standard output is set to be binary, so that "\r\n" at the +ends of output lines that are copied from the input is not converted to +"\r\r\n" by the C I/O library. This means that any messages written to the +standard output must end with "\r\n". For all other operating systems, and +for all messages to the standard error stream, "\n" is used. </P> <br><a name="SEC9" href="#TOC1">OPTIONS COMPATIBILITY</a><br> <P> @@ -806,9 +874,9 @@ as in the GNU <b>grep</b> program. Any long option of the form <b>--file-offsets</b>, <b>--heap-limit</b>, <b>--include-dir</b>, <b>--line-offsets</b>, <b>--locale</b>, <b>--match-limit</b>, <b>-M</b>, <b>--multiline</b>, <b>-N</b>, <b>--newline</b>, <b>--om-separator</b>, -<b>--output</b>, <b>-u</b>, and <b>--utf-8</b> options are specific to -<b>pcre2grep</b>, as is the use of the <b>--only-matching</b> option with a -capturing parentheses number. +<b>--output</b>, <b>-u</b>, <b>--utf</b>, <b>-U</b>, and <b>--utf-allow-invalid</b> +options are specific to <b>pcre2grep</b>, as is the use of the +<b>--only-matching</b> option with a capturing parentheses number. </P> <P> Although most of the common options work the same way, a few are different in @@ -868,12 +936,36 @@ documentation for details). Numbered callouts are ignored by <b>pcre2grep</b>; only callouts with string arguments are useful. </P> <br><b> +Echoing a specific string +</b><br> +<P> +Starting the callout string with a pipe character invokes an echoing facility +that avoids calling an external program or script. This facility is always +available, provided that callouts were not completely disabled when +<b>pcre2grep</b> was built. The rest of the callout string is processed as a +zero-terminated string, which means it should not contain any internal binary +zeros. It is written to the output, having first been passed through the same +escape processing as text from the <b>--output</b> (<b>-O</b>) option (see +above). However, $0 cannot be used to insert a matched substring because the +match is still in progress. Instead, the single character '0' is inserted. Any +syntax errors in the string (for example, a dollar not followed by another +character) causes the callout to be ignored. No terminator is added to the +output string, so if you want a newline, you must include it explicitly using +the escape $n. For example: +<pre> + pcre2grep '(.)(..(.))(?C"|[$1] [$2] [$3]$n")' <some file> +</pre> +Matching continues normally after the string is output. If you want to see only +the callout output but not any output from an actual match, you should end the +pattern with (*FAIL). +</P> +<br><b> Calling external programs or scripts </b><br> <P> This facility can be independently disabled when <b>pcre2grep</b> is built. It is supported for Windows, where a call to <b>_spawnvp()</b> is used, for VMS, -where <b>lib$spawn()</b> is used, and for any other Unix-like environment where +where <b>lib$spawn()</b> is used, and for any Unix-like environment where <b>fork()</b> and <b>execv()</b> are available. </P> <P> @@ -885,14 +977,11 @@ arguments: executable_name|arg1|arg2|... </pre> Any substring (including the executable name) may contain escape sequences -started by a dollar character: $<digits> or ${<digits>} is replaced by the -captured substring of the given decimal number, which must be greater than -zero. If the number is greater than the number of capturing substrings, or if -the capture is unset, the replacement is empty. -</P> -<P> -Any other character is substituted by itself. In particular, $$ is replaced by -a single dollar and $| is replaced by a pipe character. Here is an example: +started by a dollar character. These are the same as for the <b>--output</b> +(<b>-O</b>) option documented above, except that $0 cannot insert the matched +string because the match is still in progress. Instead, the character '0' +is inserted. If you need a literal dollar or pipe character in any +substring, use $$ or $| respectively. Here is an example: <pre> echo -e "abcde\n12345" | pcre2grep \ '(?x)(.)(..(.)) @@ -905,28 +994,14 @@ a single dollar and $| is replaced by a pipe character. Here is an example: Arg1: [1] [234] [4] Arg2: |1| () 12345 </pre> -The parameters for the system call that is used to run the -program or script are zero-terminated strings. This means that binary zero -characters in the callout argument will cause premature termination of their -substrings, and therefore should not be present. Any syntax errors in the -string (for example, a dollar not followed by another character) cause the -callout to be ignored. If running the program fails for any reason (including -the non-existence of the executable), a local matching failure occurs and the -matcher backtracks in the normal way. -</P> -<br><b> -Echoing a specific string -</b><br> -<P> -This facility is always available, provided that callouts were not completely -disabled when <b>pcre2grep</b> was built. If the callout string starts with a -pipe (vertical bar) character, the rest of the string is written to the output, -having been passed through the same escape processing as text from the --output -option. This provides a simple echoing facility that avoids calling an external -program or script. No terminator is added to the string, so if you want a -newline, you must include it explicitly. Matching continues normally after the -string is output. If you want to see only the callout output but not any output -from an actual match, you should end the relevant pattern with (*FAIL). +The parameters for the system call that is used to run the program or script +are zero-terminated strings. This means that binary zero characters in the +callout argument will cause premature termination of their substrings, and +therefore should not be present. Any syntax errors in the string (for example, +a dollar not followed by another character) causes the callout to be ignored. +If running the program fails for any reason (including the non-existence of the +executable), a local matching failure occurs and the matcher backtracks in the +normal way. </P> <br><a name="SEC12" href="#TOC1">MATCHING ERRORS</a><br> <P> @@ -958,7 +1033,8 @@ because VMS does not distinguish between exit(0) and exit(1). </P> <br><a name="SEC14" href="#TOC1">SEE ALSO</a><br> <P> -<b>pcre2pattern</b>(3), <b>pcre2syntax</b>(3), <b>pcre2callout</b>(3). +<b>pcre2pattern</b>(3), <b>pcre2syntax</b>(3), <b>pcre2callout</b>(3), +<b>pcre2unicode</b>(3). </P> <br><a name="SEC15" href="#TOC1">AUTHOR</a><br> <P> @@ -971,9 +1047,9 @@ Cambridge, England. </P> <br><a name="SEC16" href="#TOC1">REVISION</a><br> <P> -Last updated: 24 November 2018 +Last updated: 04 October 2020 <br> -Copyright © 1997-2018 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2jit.html b/dist2/doc/html/pcre2jit.html index cb4eb88e..423dfd83 100644 --- a/dist2/doc/html/pcre2jit.html +++ b/dist2/doc/html/pcre2jit.html @@ -90,7 +90,7 @@ or a negative error code. There is a limit to the size of pattern that JIT supports, imposed by the size of machine stack that it uses. The exact rules are not documented because they may change at any time, in particular, when new optimizations are introduced. -If a pattern is too big, a call to \fBpcre2_jit_compile()\fB returns +If a pattern is too big, a call to <b>pcre2_jit_compile()</b> returns PCRE2_ERROR_NOMEMORY. </P> <P> @@ -147,25 +147,29 @@ pattern. </P> <br><a name="SEC4" href="#TOC1">MATCHING SUBJECTS CONTAINING INVALID UTF</a><br> <P> -When a pattern is compiled with the PCRE2_UTF option, the interpretive matching -function expects its subject string to be a valid sequence of UTF code units. -If it is not, the result is undefined. This is also true by default of matching -via JIT. However, if the option PCRE2_JIT_INVALID_UTF is passed to -<b>pcre2_jit_compile()</b>, code that can process a subject containing invalid -UTF is compiled. +When a pattern is compiled with the PCRE2_UTF option, subject strings are +normally expected to be a valid sequence of UTF code units. By default, this is +checked at the start of matching and an error is generated if invalid UTF is +detected. The PCRE2_NO_UTF_CHECK option can be passed to <b>pcre2_match()</b> to +skip the check (for improved performance) if you are sure that a subject string +is valid. If this option is used with an invalid string, the result is +undefined. </P> <P> -In this mode, an invalid code unit sequence never matches any pattern item. It -does not match dot, it does not match \p{Any}, it does not even match negative -items such as [^X]. A lookbehind assertion fails if it encounters an invalid -sequence while moving the current point backwards. In other words, an invalid -UTF code unit sequence acts as a barrier which no match can cross. Reaching an -invalid sequence causes an immediate backtrack. +However, a way of running matches on strings that may contain invalid UTF +sequences is available. Calling <b>pcre2_compile()</b> with the +PCRE2_MATCH_INVALID_UTF option has two effects: it tells the interpreter in +<b>pcre2_match()</b> to support invalid UTF, and, if <b>pcre2_jit_compile()</b> +is called, the compiled JIT code also supports invalid UTF. Details of how this +support works, in both the JIT and the interpretive cases, is given in the +<a href="pcre2unicode.html"><b>pcre2unicode</b></a> +documentation. </P> <P> -Using this option, an application can run matches in arbitrary data, knowing -that any matched strings that are returned will be valid UTF. This can be -useful when searching for text in executable or other binary files. +There is also an obsolete option for <b>pcre2_jit_compile()</b> called +PCRE2_JIT_INVALID_UTF, which currently exists only for backward compatibility. +It is superseded by the <b>pcre2_compile()</b> option PCRE2_MATCH_INVALID_UTF +and should no longer be used. It may be removed in future. </P> <br><a name="SEC5" href="#TOC1">UNSUPPORTED OPTIONS AND PATTERN ITEMS</a><br> <P> @@ -335,12 +339,12 @@ stack through the JIT callback function. You can free a JIT stack at any time, as long as it will not be used by <b>pcre2_match()</b> again. When you assign the stack to a match context, only a pointer is set. There is no reference counting or any other magic. You can free -compiled patterns, contexts, and stacks in any order, anytime. Just \fIdo -not\fP call <b>pcre2_match()</b> with a match context pointing to an already -freed stack, as that will cause SEGFAULT. (Also, do not free a stack currently -used by <b>pcre2_match()</b> in another thread). You can also replace the stack -in a context at any time when it is not in use. You should free the previous -stack before assigning a replacement. +compiled patterns, contexts, and stacks in any order, anytime. +Just <i>do not</i> call <b>pcre2_match()</b> with a match context pointing to an +already freed stack, as that will cause SEGFAULT. (Also, do not free a stack +currently used by <b>pcre2_match()</b> in another thread). You can also replace +the stack in a context at any time when it is not in use. You should free the +previous stack before assigning a replacement. </P> <P> (5) Should I allocate/free a stack every time before/after calling @@ -461,7 +465,7 @@ Cambridge, England. </P> <br><a name="SEC14" href="#TOC1">REVISION</a><br> <P> -Last updated: 06 March 2019 +Last updated: 23 May 2019 <br> Copyright © 1997-2019 University of Cambridge. <br> diff --git a/dist2/doc/html/pcre2matching.html b/dist2/doc/html/pcre2matching.html index 3e32cb57..4b71c8f8 100644 --- a/dist2/doc/html/pcre2matching.html +++ b/dist2/doc/html/pcre2matching.html @@ -188,6 +188,10 @@ code unit) at a time, for all active paths through the tree. 9. Except for (*FAIL), the backtracking control verbs such as (*PRUNE) are not supported. (*FAIL) is supported, and behaves like a failing negative assertion. </P> +<P> +10. The PCRE2_MATCH_INVALID_UTF option for <b>pcre2_compile()</b> is not +supported by <b>pcre2_dfa_match()</b>. +</P> <br><a name="SEC5" href="#TOC1">ADVANTAGES OF THE ALTERNATIVE ALGORITHM</a><br> <P> Using the alternative matching algorithm provides the following advantages: @@ -219,7 +223,8 @@ because it has to search for all possible matches, but is also because it is less susceptible to optimization. </P> <P> -2. Capturing parentheses, backreferences, and script runs are not supported. +2. Capturing parentheses, backreferences, script runs, and matching within +invalid UTF string are not supported. </P> <P> 3. Although atomic groups are supported, their use does not provide the @@ -236,9 +241,9 @@ Cambridge, England. </P> <br><a name="SEC8" href="#TOC1">REVISION</a><br> <P> -Last updated: 10 October 2018 +Last updated: 23 May 2019 <br> -Copyright © 1997-2018 University of Cambridge. +Copyright © 1997-2019 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2partial.html b/dist2/doc/html/pcre2partial.html index 4e156b81..bb73b1de 100644 --- a/dist2/doc/html/pcre2partial.html +++ b/dist2/doc/html/pcre2partial.html @@ -14,78 +14,123 @@ please consult the man page, in case the conversion went wrong. <br> <ul> <li><a name="TOC1" href="#SEC1">PARTIAL MATCHING IN PCRE2</a> -<li><a name="TOC2" href="#SEC2">PARTIAL MATCHING USING pcre2_match()</a> -<li><a name="TOC3" href="#SEC3">PARTIAL MATCHING USING pcre2_dfa_match()</a> -<li><a name="TOC4" href="#SEC4">PARTIAL MATCHING AND WORD BOUNDARIES</a> -<li><a name="TOC5" href="#SEC5">EXAMPLE OF PARTIAL MATCHING USING PCRE2TEST</a> +<li><a name="TOC2" href="#SEC2">REQUIREMENTS FOR A PARTIAL MATCH</a> +<li><a name="TOC3" href="#SEC3">PARTIAL MATCHING USING pcre2_match()</a> +<li><a name="TOC4" href="#SEC4">MULTI-SEGMENT MATCHING WITH pcre2_match()</a> +<li><a name="TOC5" href="#SEC5">PARTIAL MATCHING USING pcre2_dfa_match()</a> <li><a name="TOC6" href="#SEC6">MULTI-SEGMENT MATCHING WITH pcre2_dfa_match()</a> -<li><a name="TOC7" href="#SEC7">MULTI-SEGMENT MATCHING WITH pcre2_match()</a> -<li><a name="TOC8" href="#SEC8">ISSUES WITH MULTI-SEGMENT MATCHING</a> -<li><a name="TOC9" href="#SEC9">AUTHOR</a> -<li><a name="TOC10" href="#SEC10">REVISION</a> +<li><a name="TOC7" href="#SEC7">AUTHOR</a> +<li><a name="TOC8" href="#SEC8">REVISION</a> </ul> <br><a name="SEC1" href="#TOC1">PARTIAL MATCHING IN PCRE2</a><br> <P> -In normal use of PCRE2, if the subject string that is passed to a matching -function matches as far as it goes, but is too short to match the entire -pattern, PCRE2_ERROR_NOMATCH is returned. There are circumstances where it -might be helpful to distinguish this case from other cases in which there is no -match. +In normal use of PCRE2, if there is a match up to the end of a subject string, +but more characters are needed to match the entire pattern, PCRE2_ERROR_NOMATCH +is returned, just like any other failing match. There are circumstances where +it might be helpful to distinguish this "partial match" case. </P> <P> -Consider, for example, an application where a human is required to type in data -for a field with specific formatting requirements. An example might be a date -in the form <i>ddmmmyy</i>, defined by this pattern: -<pre> - ^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$ -</pre> -If the application sees the user's keystrokes one by one, and can check that -what has been typed so far is potentially valid, it is able to raise an error -as soon as a mistake is made, by beeping and not reflecting the character that -has been typed, for example. This immediate feedback is likely to be a better -user interface than a check that is delayed until the entire string has been -entered. Partial matching can also be useful when the subject string is very -long and is not all available at once. +One example is an application where the subject string is very long, and not +all available at once. The requirement here is to be able to do the matching +segment by segment, but special action is needed when a matched substring spans +the boundary between two segments. </P> <P> -PCRE2 supports partial matching by means of the PCRE2_PARTIAL_SOFT and -PCRE2_PARTIAL_HARD options, which can be set when calling a matching function. -The difference between the two options is whether or not a partial match is -preferred to an alternative complete match, though the details differ between -the two types of matching function. If both options are set, PCRE2_PARTIAL_HARD -takes precedence. +Another example is checking a user input string as it is typed, to ensure that +it conforms to a required format. Invalid characters can be immediately +diagnosed and rejected, giving instant feedback. </P> <P> -If you want to use partial matching with just-in-time optimized code, you must -call <b>pcre2_jit_compile()</b> with one or both of these options: +Partial matching is a PCRE2-specific feature; it is not Perl-compatible. It is +requested by setting one of the PCRE2_PARTIAL_HARD or PCRE2_PARTIAL_SOFT +options when calling a matching function. The difference between the two +options is whether or not a partial match is preferred to an alternative +complete match, though the details differ between the two types of matching +function. If both options are set, PCRE2_PARTIAL_HARD takes precedence. +</P> +<P> +If you want to use partial matching with just-in-time optimized code, as well +as setting a partial match option for the matching function, you must also call +<b>pcre2_jit_compile()</b> with one or both of these options: <pre> - PCRE2_JIT_PARTIAL_SOFT PCRE2_JIT_PARTIAL_HARD + PCRE2_JIT_PARTIAL_SOFT </pre> PCRE2_JIT_COMPLETE should also be set if you are going to run non-partial -matches on the same pattern. If the appropriate JIT mode has not been compiled, -interpretive matching code is used. +matches on the same pattern. Separate code is compiled for each mode. If the +appropriate JIT mode has not been compiled, interpretive matching code is used. </P> <P> Setting a partial matching option disables two of PCRE2's standard -optimizations. PCRE2 remembers the last literal code unit in a pattern, and -abandons matching immediately if it is not present in the subject string. This -optimization cannot be used for a subject string that might match only -partially. PCRE2 also knows the minimum length of a matching string, and does +optimization hints. PCRE2 remembers the last literal code unit in a pattern, +and abandons matching immediately if it is not present in the subject string. +This optimization cannot be used for a subject string that might match only +partially. PCRE2 also remembers a minimum length of a matching string, and does not bother to run the matching function on shorter strings. This optimization is also disabled for partial matching. </P> -<br><a name="SEC2" href="#TOC1">PARTIAL MATCHING USING pcre2_match()</a><br> +<br><a name="SEC2" href="#TOC1">REQUIREMENTS FOR A PARTIAL MATCH</a><br> +<P> +A possible partial match occurs during matching when the end of the subject +string is reached successfully, but either more characters are needed to +complete the match, or the addition of more characters might change what is +matched. +</P> <P> -A partial match occurs during a call to <b>pcre2_match()</b> when the end of the -subject string is reached successfully, but matching cannot continue because -more characters are needed. However, at least one character in the subject must -have been inspected. This character need not form part of the final matched -string; lookbehind assertions and the \K escape sequence provide ways of -inspecting characters before the start of a matched string. The requirement for -inspecting at least one character exists because an empty string can always be -matched; without such a restriction there would always be a partial match of an -empty string at the end of the subject. +Example 1: if the pattern is /abc/ and the subject is "ab", more characters are +definitely needed to complete a match. In this case both hard and soft matching +options yield a partial match. +</P> +<P> +Example 2: if the pattern is /ab+/ and the subject is "ab", a complete match +can be found, but the addition of more characters might change what is +matched. In this case, only PCRE2_PARTIAL_HARD returns a partial match; +PCRE2_PARTIAL_SOFT returns the complete match. +</P> +<P> +On reaching the end of the subject, when PCRE2_PARTIAL_HARD is set, if the next +pattern item is \z, \Z, \b, \B, or $ there is always a partial match. +Otherwise, for both options, the next pattern item must be one that inspects a +character, and at least one of the following must be true: +</P> +<P> +(1) At least one character has already been inspected. An inspected character +need not form part of the final matched string; lookbehind assertions and the +\K escape sequence provide ways of inspecting characters before the start of a +matched string. +</P> +<P> +(2) The pattern contains one or more lookbehind assertions. This condition +exists in case there is a lookbehind that inspects characters before the start +of the match. +</P> +<P> +(3) There is a special case when the whole pattern can match an empty string. +When the starting point is at the end of the subject, the empty string match is +a possibility, and if PCRE2_PARTIAL_SOFT is set and neither of the above +conditions is true, it is returned. However, because adding more characters +might result in a non-empty match, PCRE2_PARTIAL_HARD returns a partial match, +which in this case means "there is going to be a match at this point, but until +some more characters are added, we do not know if it will be an empty string or +something longer". +</P> +<br><a name="SEC3" href="#TOC1">PARTIAL MATCHING USING pcre2_match()</a><br> +<P> +When a partial matching option is set, the result of calling +<b>pcre2_match()</b> can be one of the following: +</P> +<P> +<b>A successful match</b> +A complete match has been found, starting and ending within this subject. +</P> +<P> +<b>PCRE2_ERROR_NOMATCH</b> +No match can start anywhere in this subject. +</P> +<P> +<b>PCRE2_ERROR_PARTIAL</b> +Adding more characters may result in a complete match that uses one or more +characters from the end of this subject. </P> <P> When a partial match is returned, the first two elements in the ovector point @@ -103,54 +148,42 @@ these characters are needed for a subsequent re-match with additional characters. </P> <P> -What happens when a partial match is identified depends on which of the two -partial matching options are set. -</P> -<br><b> -PCRE2_PARTIAL_SOFT WITH pcre2_match() -</b><br> -<P> -If PCRE2_PARTIAL_SOFT is set when <b>pcre2_match()</b> identifies a partial -match, the partial match is remembered, but matching continues as normal, and -other alternatives in the pattern are tried. If no complete match can be found, -PCRE2_ERROR_PARTIAL is returned instead of PCRE2_ERROR_NOMATCH. -</P> -<P> -This option is "soft" because it prefers a complete match over a partial match. -All the various matching items in a pattern behave as if the subject string is -potentially complete. For example, \z, \Z, and $ match at the end of the -subject, as normal, and for \b and \B the end of the subject is treated as a -non-alphanumeric. -</P> -<P> If there is more than one partial match, the first one that was found provides the data that is returned. Consider this pattern: <pre> /123\w+X|dogY/ </pre> -If this is matched against the subject string "abc123dog", both -alternatives fail to match, but the end of the subject is reached during -matching, so PCRE2_ERROR_PARTIAL is returned. The offsets are set to 3 and 9, -identifying "123dog" as the first partial match that was found. (In this -example, there are two partial matches, because "dog" on its own partially -matches the second alternative.) +If this is matched against the subject string "abc123dog", both alternatives +fail to match, but the end of the subject is reached during matching, so +PCRE2_ERROR_PARTIAL is returned. The offsets are set to 3 and 9, identifying +"123dog" as the first partial match. (In this example, there are two partial +matches, because "dog" on its own partially matches the second alternative.) </P> <br><b> -PCRE2_PARTIAL_HARD WITH pcre2_match() +How a partial match is processed by pcre2_match() </b><br> <P> -If PCRE2_PARTIAL_HARD is set for <b>pcre2_match()</b>, PCRE2_ERROR_PARTIAL is -returned as soon as a partial match is found, without continuing to search for -possible complete matches. This option is "hard" because it prefers an earlier -partial match over a later complete match. For this reason, the assumption is -made that the end of the supplied subject string may not be the true end of the -available data, and so, if \z, \Z, \b, \B, or $ are encountered at the end -of the subject, the result is PCRE2_ERROR_PARTIAL, provided that at least one -character in the subject has been inspected. +What happens when a partial match is identified depends on which of the two +partial matching options is set. +</P> +<P> +If PCRE2_PARTIAL_HARD is set, PCRE2_ERROR_PARTIAL is returned as soon as a +partial match is found, without continuing to search for possible complete +matches. This option is "hard" because it prefers an earlier partial match over +a later complete match. For this reason, the assumption is made that the end of +the supplied subject string is not the true end of the available data, which is +why \z, \Z, \b, \B, and $ always give a partial match. +</P> +<P> +If PCRE2_PARTIAL_SOFT is set, the partial match is remembered, but matching +continues as normal, and other alternatives in the pattern are tried. If no +complete match can be found, PCRE2_ERROR_PARTIAL is returned instead of +PCRE2_ERROR_NOMATCH. This option is "soft" because it prefers a complete match +over a partial match. All the various matching items in a pattern behave as if +the subject string is potentially complete; \z, \Z, and $ match at the end of +the subject, as normal, and for \b and \B the end of the subject is treated +as a non-alphanumeric. </P> -<br><b> -Comparing hard and soft partial matching -</b><br> <P> The difference between the two partial matching options can be illustrated by a pattern such as: @@ -175,245 +208,167 @@ to follow this explanation by thinking of the two patterns like this: The second pattern will never match "dogsbody", because it will always find the shorter match first. </P> -<br><a name="SEC3" href="#TOC1">PARTIAL MATCHING USING pcre2_dfa_match()</a><br> -<P> -The DFA functions move along the subject string character by character, without -backtracking, searching for all possible matches simultaneously. If the end of -the subject is reached before the end of the pattern, there is the possibility -of a partial match, again provided that at least one character has been -inspected. -</P> -<P> -When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if there -have been no complete matches. Otherwise, the complete matches are returned. -However, if PCRE2_PARTIAL_HARD is set, a partial match takes precedence over -any complete matches. The portion of the string that was matched when the -longest partial match was found is set as the first matching string. -</P> -<P> -Because the DFA functions always search for all possible matches, and there is -no difference between greedy and ungreedy repetition, their behaviour is -different from the standard functions when PCRE2_PARTIAL_HARD is set. Consider -the string "dog" matched against the ungreedy pattern shown above: -<pre> - /dog(sbody)??/ -</pre> -Whereas the standard function stops as soon as it finds the complete match for -"dog", the DFA function also finds the partial match for "dogsbody", and so -returns that when PCRE2_PARTIAL_HARD is set. -</P> -<br><a name="SEC4" href="#TOC1">PARTIAL MATCHING AND WORD BOUNDARIES</a><br> +<br><b> +Example of partial matching using pcre2test +</b><br> <P> -If a pattern ends with one of sequences \b or \B, which test for word -boundaries, partial matching with PCRE2_PARTIAL_SOFT can give counter-intuitive -results. Consider this pattern: -<pre> - /\bcat\b/ -</pre> -This matches "cat", provided there is a word boundary at either end. If the -subject string is "the cat", the comparison of the final "t" with a following -character cannot take place, so a partial match is found. However, normal -matching carries on, and \b matches at the end of the subject when the last -character is a letter, so a complete match is found. The result, therefore, is -<i>not</i> PCRE2_ERROR_PARTIAL. Using PCRE2_PARTIAL_HARD in this case does yield -PCRE2_ERROR_PARTIAL, because then the partial match takes precedence. -</P> -<br><a name="SEC5" href="#TOC1">EXAMPLE OF PARTIAL MATCHING USING PCRE2TEST</a><br> -<P> -If the <b>partial_soft</b> (or <b>ps</b>) modifier is present on a -<b>pcre2test</b> data line, the PCRE2_PARTIAL_SOFT option is used for the match. -Here is a run of <b>pcre2test</b> that uses the date example quoted above: +The <b>pcre2test</b> data modifiers <b>partial_hard</b> (or <b>ph</b>) and +<b>partial_soft</b> (or <b>ps</b>) set PCRE2_PARTIAL_HARD and PCRE2_PARTIAL_SOFT, +respectively, when calling <b>pcre2_match()</b>. Here is a run of +<b>pcre2test</b> using a pattern that matches the whole subject in the form of a +date: <pre> re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ - data> 25jun04\=ps - 0: 25jun04 - 1: jun - data> 25dec3\=ps + data> 25dec3\=ph Partial match: 23dec3 - data> 3ju\=ps + data> 3ju\=ph Partial match: 3ju - data> 3juj\=ps - No match - data> j\=ps + data> 3juj\=ph No match </pre> -The first data string is matched completely, so <b>pcre2test</b> shows the -matched substrings. The remaining four strings do not match the complete -pattern, but the first two are partial matches. Similar output is obtained -if DFA matching is used. -</P> -<P> -If the <b>partial_hard</b> (or <b>ph</b>) modifier is present on a -<b>pcre2test</b> data line, the PCRE2_PARTIAL_HARD option is set for the match. -</P> -<br><a name="SEC6" href="#TOC1">MULTI-SEGMENT MATCHING WITH pcre2_dfa_match()</a><br> -<P> -When a partial match has been found using a DFA matching function, it is -possible to continue the match by providing additional subject data and calling -the function again with the same compiled regular expression, this time setting -the PCRE2_DFA_RESTART option. You must pass the same working space as before, -because this is where details of the previous partial match are stored. Here is -an example using <b>pcre2test</b>: +This example gives the same results for both hard and soft partial matching +options. Here is an example where there is a difference: <pre> re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ - data> 23ja\=dfa,ps - Partial match: 23ja - data> n05\=dfa,dfa_restart - 0: n05 + data> 25jun04\=ps + 0: 25jun04 + 1: jun + data> 25jun04\=ph + Partial match: 25jun04 </pre> -The first call has "23ja" as the subject, and requests partial matching; the -second call has "n05" as the subject for the continued (restarted) match. -Notice that when the match is complete, only the last part is shown; PCRE2 does -not retain the previously partially-matched string. It is up to the calling -program to do that if it needs to. -</P> -<P> -That means that, for an unanchored pattern, if a continued match fails, it is -not possible to try again at a new starting point. All this facility is capable -of doing is continuing with the previous match attempt. In the previous -example, if the second set of data is "ug23" the result is no match, even -though there would be a match for "aug23" if the entire string were given at -once. Depending on the application, this may or may not be what you want. -The only way to allow for starting again at the next character is to retain the -matched part of the subject and try a new complete match. +With PCRE2_PARTIAL_SOFT, the subject is matched completely. For +PCRE2_PARTIAL_HARD, however, the subject is assumed not to be complete, so +there is only a partial match. </P> +<br><a name="SEC4" href="#TOC1">MULTI-SEGMENT MATCHING WITH pcre2_match()</a><br> <P> -You can set the PCRE2_PARTIAL_SOFT or PCRE2_PARTIAL_HARD options with -PCRE2_DFA_RESTART to continue partial matching over multiple segments. This -facility can be used to pass very long subject strings to the DFA matching -functions. +PCRE was not originally designed with multi-segment matching in mind. However, +over time, features (including partial matching) that make multi-segment +matching possible have been added. A very long string can be searched segment +by segment by calling <b>pcre2_match()</b> repeatedly, with the aim of achieving +the same results that would happen if the entire string was available for +searching all the time. Normally, the strings that are being sought are much +shorter than each individual segment, and are in the middle of very long +strings, so the pattern is normally not anchored. </P> -<br><a name="SEC7" href="#TOC1">MULTI-SEGMENT MATCHING WITH pcre2_match()</a><br> <P> -Unlike the DFA function, it is not possible to restart the previous match with -a new segment of data when using <b>pcre2_match()</b>. Instead, new data must be -added to the previous subject string, and the entire match re-run, starting -from the point where the partial match occurred. Earlier data can be discarded. +Special logic must be implemented to handle a matched substring that spans a +segment boundary. PCRE2_PARTIAL_HARD should be used, because it returns a +partial match at the end of a segment whenever there is the possibility of +changing the match by adding more characters. The PCRE2_NOTBOL option should +also be set for all but the first segment. </P> <P> -It is best to use PCRE2_PARTIAL_HARD in this situation, because it does not -treat the end of a segment as the end of the subject when matching \z, \Z, -\b, \B, and $. Consider an unanchored pattern that matches dates: +When a partial match occurs, the next segment must be added to the current +subject and the match re-run, using the <i>startoffset</i> argument of +<b>pcre2_match()</b> to begin at the point where the partial match started. +For example: <pre> re> /\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d/ - data> The date is 23ja\=ph + data> ...the date is 23ja\=ph Partial match: 23ja + data> ...the date is 23jan19 and on that day...\=offset=15 + 0: 23jan19 + 1: jan </pre> -At this stage, an application could discard the text preceding "23ja", add on -text from the next segment, and call the matching function again. Unlike the -DFA matching function, the entire matching string must always be available, -and the complete matching process occurs for each call, so more memory and more -processing time is needed. -</P> -<br><a name="SEC8" href="#TOC1">ISSUES WITH MULTI-SEGMENT MATCHING</a><br> -<P> -Certain types of pattern may give problems with multi-segment matching, -whichever matching function is used. +Note the use of the <b>offset</b> modifier to start the new match where the +partial match was found. In this example, the next segment was added to the one +in which the partial match was found. This is the most straightforward +approach, typically using a memory buffer that is twice the size of each +segment. After a partial match, the first half of the buffer is discarded, the +second half is moved to the start of the buffer, and a new segment is added +before repeating the match as in the example above. After a no match, the +entire buffer can be discarded. +</P> +<P> +If there are memory constraints, you may want to discard text that precedes a +partial match before adding the next segment. Unfortunately, this is not at +present straightforward. In cases such as the above, where the pattern does not +contain any lookbehinds, it is sufficient to retain only the partially matched +substring. However, if the pattern contains a lookbehind assertion, characters +that precede the start of the partial match may have been inspected during the +matching process. When <b>pcre2test</b> displays a partial match, it indicates +these characters with '<' if the <b>allusedtext</b> modifier is set: +<pre> + re> "(?<=123)abc" + data> xx123ab\=ph,allusedtext + Partial match: 123ab + <<< +</pre> +However, the <b>allusedtext</b> modifier is not available for JIT matching, +because JIT matching does not record the first (or last) consulted characters. +For this reason, this information is not available via the API. It is therefore +not possible in general to obtain the exact number of characters that must be +retained in order to get the right match result. If you cannot retain the +entire segment, you must find some heuristic way of choosing. </P> <P> -1. If the pattern contains a test for the beginning of a line, you need to pass -the PCRE2_NOTBOL option when the subject string for any call does start at the -beginning of a line. There is also a PCRE2_NOTEOL option, but in practice when -doing multi-segment matching you should be using PCRE2_PARTIAL_HARD, which -includes the effect of PCRE2_NOTEOL. +If you know the approximate length of the matching substrings, you can use that +to decide how much text to retain. The only lookbehind information that is +currently available via the API is the length of the longest individual +lookbehind in a pattern, but this can be misleading if there are nested +lookbehinds. The value returned by calling <b>pcre2_pattern_info()</b> with the +PCRE2_INFO_MAXLOOKBEHIND option is the maximum number of characters (not code +units) that any individual lookbehind moves back when it is processed. A +pattern such as "(?<=(?<!b)a)" has a maximum lookbehind value of one, but +inspects two characters before its starting point. </P> <P> -2. If a pattern contains a lookbehind assertion, characters that precede the -start of the partial match may have been inspected during the matching process. -When using <b>pcre2_match()</b>, sufficient characters must be retained for the -next match attempt. You can ensure that enough characters are retained by doing -the following: +In a non-UTF or a 32-bit case, moving back is just a subtraction, but in +UTF-8 or UTF-16 you have to count characters while moving back through the code +units. </P> +<br><a name="SEC5" href="#TOC1">PARTIAL MATCHING USING pcre2_dfa_match()</a><br> <P> -Before doing any matching, find the length of the longest lookbehind in the -pattern by calling <b>pcre2_pattern_info()</b> with the PCRE2_INFO_MAXLOOKBEHIND -option. Note that the resulting count is in characters, not code units. After a -partial match, moving back from the ovector[0] offset in the subject by the -number of characters given for the maximum lookbehind gets you to the earliest -character that must be retained. In a non-UTF or a 32-bit situation, moving -back is just a subtraction, but in UTF-8 or UTF-16 you have to count characters -while moving back through the code units. +The DFA function moves along the subject string character by character, without +backtracking, searching for all possible matches simultaneously. If the end of +the subject is reached before the end of the pattern, there is the possibility +of a partial match. </P> <P> -Characters before the point you have now reached can be discarded, and after -the next segment has been added to what is retained, you should run the next -match with the <b>startoffset</b> argument set so that the match begins at the -same point as before. +When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if there +have been no complete matches. Otherwise, the complete matches are returned. +If PCRE2_PARTIAL_HARD is set, a partial match takes precedence over any +complete matches. The portion of the string that was matched when the longest +partial match was found is set as the first matching string. </P> <P> -For example, if the pattern "(?<=123)abc" is partially matched against the -string "xx123ab", the ovector offsets are 5 and 7 ("ab"). The maximum -lookbehind count is 3, so all characters before offset 2 can be discarded. The -value of <b>startoffset</b> for the next match should be 3. When <b>pcre2test</b> -displays a partial match, it indicates the lookbehind characters with '<' -characters: +Because the DFA function always searches for all possible matches, and there is +no difference between greedy and ungreedy repetition, its behaviour is +different from the <b>pcre2_match()</b>. Consider the string "dog" matched +against this ungreedy pattern: <pre> - re> "(?<=123)abc" - data> xx123ab\=ph - Partial match: 123ab - <<< -</PRE> + /dog(sbody)??/ +</pre> +Whereas the standard function stops as soon as it finds the complete match for +"dog", the DFA function also finds the partial match for "dogsbody", and so +returns that when PCRE2_PARTIAL_HARD is set. </P> +<br><a name="SEC6" href="#TOC1">MULTI-SEGMENT MATCHING WITH pcre2_dfa_match()</a><br> <P> -3. Because a partial match must always contain at least one character, what -might be considered a partial match of an empty string actually gives a "no -match" result. For example: -<pre> - re> /c(?<=abc)x/ - data> ab\=ps - No match -</pre> -If the next segment begins "cx", a match should be found, but this will only -happen if characters from the previous segment are retained. For this reason, a -"no match" result should be interpreted as "partial match of an empty string" -when the pattern contains lookbehinds. -</P> -<P> -4. Matching a subject string that is split into multiple segments may not -always produce exactly the same result as matching over one single long string, -especially when PCRE2_PARTIAL_SOFT is used. The section "Partial Matching and -Word Boundaries" above describes an issue that arises if the pattern ends with -\b or \B. Another kind of difference may occur when there are multiple -matching possibilities, because (for PCRE2_PARTIAL_SOFT) a partial match result -is given only when there are no completed matches. This means that as soon as -the shortest match has been found, continuation to a new subject segment is no -longer possible. Consider this <b>pcre2test</b> example: -<pre> - re> /dog(sbody)?/ - data> dogsb\=ps - 0: dog - data> do\=ps,dfa - Partial match: do - data> gsb\=ps,dfa,dfa_restart - 0: g - data> dogsbody\=dfa - 0: dogsbody - 1: dog -</pre> -The first data line passes the string "dogsb" to a standard matching function, -setting the PCRE2_PARTIAL_SOFT option. Although the string is a partial match -for "dogsbody", the result is not PCRE2_ERROR_PARTIAL, because the shorter -string "dog" is a complete match. Similarly, when the subject is presented to -a DFA matching function in several parts ("do" and "gsb" being the first two) -the match stops when "dog" has been found, and it is not possible to continue. -On the other hand, if "dogsbody" is presented as a single string, a DFA -matching function finds both matches. -</P> -<P> -Because of these problems, it is best to use PCRE2_PARTIAL_HARD when matching -multi-segment data. The example above then behaves differently: +When a partial match has been found using the DFA matching function, it is +possible to continue the match by providing additional subject data and calling +the function again with the same compiled regular expression, this time setting +the PCRE2_DFA_RESTART option. You must pass the same working space as before, +because this is where details of the previous partial match are stored. You can +set the PCRE2_PARTIAL_SOFT or PCRE2_PARTIAL_HARD options with PCRE2_DFA_RESTART +to continue partial matching over multiple segments. Here is an example using +<b>pcre2test</b>: <pre> - re> /dog(sbody)?/ - data> dogsb\=ph - Partial match: dogsb - data> do\=ps,dfa - Partial match: do - data> gsb\=ph,dfa,dfa_restart - Partial match: gsb + re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ + data> 23ja\=dfa,ps + Partial match: 23ja + data> n05\=dfa,dfa_restart + 0: n05 </pre> -5. Patterns that contain alternatives at the top level which do not all start -with the same pattern item may not work as expected when PCRE2_DFA_RESTART is -used. For example, consider this pattern: +The first call has "23ja" as the subject, and requests partial matching; the +second call has "n05" as the subject for the continued (restarted) match. +Notice that when the match is complete, only the last part is shown; PCRE2 does +not retain the previously partially-matched string. It is up to the calling +program to do that if it needs to. This means that, for an unanchored pattern, +if a continued match fails, it is not possible to try again at a new starting +point. All this facility is capable of doing is continuing with the previous +match attempt. For example, consider this pattern: <pre> 1234|3789 </pre> @@ -422,30 +377,18 @@ alternative is found at offset 3. There is no partial match for the second alternative, because such a match does not start at the same point in the subject string. Attempting to continue with the string "7890" does not yield a match because only those alternatives that match at one point in the subject -are remembered. The problem arises because the start of the second alternative -matches within the first alternative. There is no problem with anchored -patterns or patterns such as: -<pre> - 1234|ABCD -</pre> -where no string can be a partial match for both alternatives. This is not a -problem if a standard matching function is used, because the entire match has -to be rerun each time: -<pre> - re> /1234|3789/ - data> ABC123\=ph - Partial match: 123 - data> 1237890 - 0: 3789 -</pre> -Of course, instead of using PCRE2_DFA_RESTART, the same technique of re-running -the entire match can also be used with the DFA matching function. Another -possibility is to work with two buffers. If a partial match at offset <i>n</i> -in the first buffer is followed by "no match" when PCRE2_DFA_RESTART is used on -the second buffer, you can then try a new match starting at offset <i>n+1</i> in -the first buffer. +are remembered. Depending on the application, this may or may not be what you +want. +</P> +<P> +If you do want to allow for starting again at the next character, one way of +doing it is to retain some or all of the segment and try a new complete match, +as described for <b>pcre2_match()</b> above. Another possibility is to work with +two buffers. If a partial match at offset <i>n</i> in the first buffer is +followed by "no match" when PCRE2_DFA_RESTART is used on the second buffer, you +can then try a new match starting at offset <i>n+1</i> in the first buffer. </P> -<br><a name="SEC9" href="#TOC1">AUTHOR</a><br> +<br><a name="SEC7" href="#TOC1">AUTHOR</a><br> <P> Philip Hazel <br> @@ -454,11 +397,11 @@ University Computing Service Cambridge, England. <br> </P> -<br><a name="SEC10" href="#TOC1">REVISION</a><br> +<br><a name="SEC8" href="#TOC1">REVISION</a><br> <P> -Last updated: 22 December 2014 +Last updated: 04 September 2019 <br> -Copyright © 1997-2014 University of Cambridge. +Copyright © 1997-2019 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2pattern.html b/dist2/doc/html/pcre2pattern.html index e6958c13..9db15b9a 100644 --- a/dist2/doc/html/pcre2pattern.html +++ b/dist2/doc/html/pcre2pattern.html @@ -33,17 +33,18 @@ please consult the man page, in case the conversion went wrong. <li><a name="TOC18" href="#SEC18">ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS</a> <li><a name="TOC19" href="#SEC19">BACKREFERENCES</a> <li><a name="TOC20" href="#SEC20">ASSERTIONS</a> -<li><a name="TOC21" href="#SEC21">SCRIPT RUNS</a> -<li><a name="TOC22" href="#SEC22">CONDITIONAL GROUPS</a> -<li><a name="TOC23" href="#SEC23">COMMENTS</a> -<li><a name="TOC24" href="#SEC24">RECURSIVE PATTERNS</a> -<li><a name="TOC25" href="#SEC25">GROUPS AS SUBROUTINES</a> -<li><a name="TOC26" href="#SEC26">ONIGURUMA SUBROUTINE SYNTAX</a> -<li><a name="TOC27" href="#SEC27">CALLOUTS</a> -<li><a name="TOC28" href="#SEC28">BACKTRACKING CONTROL</a> -<li><a name="TOC29" href="#SEC29">SEE ALSO</a> -<li><a name="TOC30" href="#SEC30">AUTHOR</a> -<li><a name="TOC31" href="#SEC31">REVISION</a> +<li><a name="TOC21" href="#SEC21">NON-ATOMIC ASSERTIONS</a> +<li><a name="TOC22" href="#SEC22">SCRIPT RUNS</a> +<li><a name="TOC23" href="#SEC23">CONDITIONAL GROUPS</a> +<li><a name="TOC24" href="#SEC24">COMMENTS</a> +<li><a name="TOC25" href="#SEC25">RECURSIVE PATTERNS</a> +<li><a name="TOC26" href="#SEC26">GROUPS AS SUBROUTINES</a> +<li><a name="TOC27" href="#SEC27">ONIGURUMA SUBROUTINE SYNTAX</a> +<li><a name="TOC28" href="#SEC28">CALLOUTS</a> +<li><a name="TOC29" href="#SEC29">BACKTRACKING CONTROL</a> +<li><a name="TOC30" href="#SEC30">SEE ALSO</a> +<li><a name="TOC31" href="#SEC31">AUTHOR</a> +<li><a name="TOC32" href="#SEC32">REVISION</a> </ul> <br><a name="SEC1" href="#TOC1">PCRE2 REGULAR EXPRESSION DETAILS</a><br> <P> @@ -91,10 +92,11 @@ single code units, or as multiple UTF-8 or UTF-16 code units. UTF-32 can be specified for the 32-bit library, in which case it constrains the character values to valid Unicode code points. To process UTF strings, PCRE2 must be built to include Unicode support (which is the default). When using UTF strings -you must either call the compiling function with the PCRE2_UTF option, or the -pattern must start with the special sequence (*UTF), which is equivalent to -setting the relevant option. How setting a UTF mode affects pattern matching is -mentioned in several places below. There is also a summary of features in the +you must either call the compiling function with one or both of the PCRE2_UTF +or PCRE2_MATCH_INVALID_UTF options, or the pattern must start with the special +sequence (*UTF), which is equivalent to setting the relevant PCRE2_UTF. How +setting a UTF mode affects pattern matching is mentioned in several places +below. There is also a summary of features in the <a href="pcre2unicode.html"><b>pcre2unicode</b></a> page. </P> @@ -112,7 +114,8 @@ Another special sequence that may appear at the start of a pattern is (*UCP). This has the same effect as setting the PCRE2_UCP option: it causes sequences such as \d and \w to use Unicode properties to determine character types, instead of recognizing only characters with codes less than 256 via a lookup -table. +table. If also causes upper/lower casing operations to use Unicode properties +for characters with code points greater than 127, even when UTF is not set. </P> <P> Some applications that allow their users to supply patterns may wish to @@ -286,8 +289,11 @@ corresponding characters in the subject. As a trivial example, the pattern The quick brown fox </pre> matches a portion of a subject string that is identical to itself. When -caseless matching is specified (the PCRE2_CASELESS option), letters are matched -independently of case. +caseless matching is specified (the PCRE2_CASELESS option or (?i) within the +pattern), letters are matched independently of case. Note that there are two +ASCII characters, K and S, that, in addition to their lower case ASCII +equivalents, are case-equivalent with Unicode U+212A (Kelvin sign) and U+017F +(long S) respectively when either PCRE2_UTF or PCRE2_UCP is set. </P> <P> The power of regular expressions comes from the ability to include wild cards, @@ -323,6 +329,20 @@ a character class the only metacharacters are: [ POSIX character class (if followed by POSIX syntax) ] terminates the character class </pre> +If a pattern is compiled with the PCRE2_EXTENDED option, most white space in +the pattern, other than in a character class, and characters between a # +outside a character class and the next newline, inclusive, are ignored. An +escaping backslash can be used to include a white space or a # character as +part of the pattern. If the PCRE2_EXTENDED_MORE option is set, the same +applies, but in addition unescaped space and horizontal tab characters are +ignored inside a character class. Note: only these two characters are ignored, +not the full set of pattern white space characters that are ignored outside a +character class. Option settings can be changed within a pattern; see the +section entitled +<a href="#internaloptions">"Internal Option Setting"</a> +below. +</P> +<P> The following sections describe the use of each of the metacharacters. </P> <br><a name="SEC5" href="#TOC1">BACKSLASH</a><br> @@ -340,16 +360,9 @@ precede a non-alphanumeric with backslash to specify that it stands for itself. In particular, if you want to match a backslash, you write \\. </P> <P> -In a UTF mode, only ASCII digits and letters have any special meaning after a -backslash. All other characters (in particular, those whose code points are -greater than 127) are treated as literals. -</P> -<P> -If a pattern is compiled with the PCRE2_EXTENDED option, most white space in -the pattern (other than in a character class), and characters between a # -outside a character class and the next newline, inclusive, are ignored. An -escaping backslash can be used to include a white space or # character as part -of the pattern. +Only ASCII digits and letters have any special meaning after a backslash. All +other characters (in particular, those whose code points are greater than 127) +are treated as literals. </P> <P> If you want to treat all characters in a sequence as literals, you can do so by @@ -428,11 +441,11 @@ There may be any number of hexadecimal digits. This syntax is from ECMAScript 6. </P> <P> -The \N{U+hhh..} escape sequence is recognized only when the PCRE2_UTF option -is set, that is, when PCRE2 is operating in a Unicode mode. Perl also uses -\N{name} to specify characters by Unicode name; PCRE2 does not support this. -Note that when \N is not followed by an opening brace (curly bracket) it has -an entirely different meaning, matching any character that is not a newline. +The \N{U+hhh..} escape sequence is recognized only when PCRE2 is operating in +UTF mode. Perl also uses \N{name} to specify characters by Unicode name; PCRE2 +does not support this. Note that when \N is not followed by an opening brace +(curly bracket) it has an entirely different meaning, matching any character +that is not a newline. </P> <P> There are some legacy applications where the escape sequence \r is expected to @@ -816,6 +829,7 @@ Caucasian_Albanian, Chakma, Cham, Cherokee, +Chorasmian, Common, Coptic, Cuneiform, @@ -823,10 +837,12 @@ Cypriot, Cyrillic, Deseret, Devanagari, +Dives_Akuru, Dogra, Duployan, Egyptian_Hieroglyphs, Elbasan, +Elymaic, Ethiopic, Georgian, Glagolitic, @@ -853,6 +869,7 @@ Kannada, Katakana, Kayah_Li, Kharoshthi, +Khitan_Small_Script, Khmer, Khojki, Khudawadi, @@ -884,10 +901,12 @@ Mro, Multani, Myanmar, Nabataean, +Nandinagari, New_Tai_Lue, Newa, Nko, Nushu, +Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungarian, @@ -939,7 +958,9 @@ Tirhuta, Ugaritic, Unknown, Vai, +Wancho, Warang_Citi, +Yezidi, Yi, Zanabazar_Square. </P> @@ -1154,8 +1175,9 @@ For example, when the pattern matches "foobar", the first substring is still set to "foo". </P> <P> -Perl documents that the use of \K within assertions is "not well defined". In -PCRE2, \K is acted upon when it occurs inside positive assertions, but is +Perl used to document that the use of \K within lookaround assertions is "not +well defined", but from version 5.32.0 Perl does not support this usage at all. +In PCRE2, \K is acted upon when it occurs inside positive assertions, but is ignored in negative assertions. Note that when a pattern such as (?=ab\K) matches, the reported start of the match can be greater than the end of the match. Using \K in a lookbehind assertion at the start of a pattern can also @@ -1360,7 +1382,7 @@ with \C in UTF-8 or UTF-16 mode means that the rest of the string may start with a malformed UTF character. This has undefined results, because PCRE2 assumes that it is matching character by character in a valid UTF string (by default it checks the subject string's validity at the start of processing -unless the PCRE2_NO_UTF_CHECK option is used). +unless the PCRE2_NO_UTF_CHECK or PCRE2_MATCH_INVALID_UTF option is used). </P> <P> An application can lock out the use of \C by setting the @@ -1432,7 +1454,10 @@ Characters in a class may be specified by their code points using \o, \x, or \N{U+hh..} in the usual way. When caseless matching is set, any letters in a class represent both their upper case and lower case versions, so for example, a caseless [aeiou] matches "A" as well as "a", and a caseless [^aeiou] does not -match "A", whereas a caseful version would. +match "A", whereas a caseful version would. Note that there are two ASCII +characters, K and S, that, in addition to their lower case ASCII equivalents, +are case-equivalent with Unicode U+212A (Kelvin sign) and U+017F (long S) +respectively when either PCRE2_UTF or PCRE2_UCP is set. </P> <P> Characters that might indicate line breaks are never treated in any special way @@ -1644,7 +1669,7 @@ that succeeds is used. If the alternatives are within a group <a href="#group">(defined below),</a> "succeeds" means matching the rest of the main pattern as well as the alternative in the group. -</P> +<a name="internaloptions"></a></P> <br><a name="SEC13" href="#TOC1">INTERNAL OPTION SETTING</a><br> <P> The settings of the PCRE2_CASELESS, PCRE2_MULTILINE, PCRE2_DOTALL, @@ -1895,12 +1920,19 @@ are permitted for groups with the same number, for example: (?|(?<AA>aa)|(?<AA>bb)) </pre> The duplicate name constraint can be disabled by setting the PCRE2_DUPNAMES -option at compile time, or by the use of (?J) within the pattern. Duplicate -names can be useful for patterns where only one instance of the named capture -group can match. Suppose you want to match the name of a weekday, either as a -3-letter abbreviation or as the full name, and in both cases you want to -extract the abbreviation. This pattern (ignoring the line breaks) does the job: +option at compile time, or by the use of (?J) within the pattern, as described +in the section entitled +<a href="#internaloptions">"Internal Option Setting"</a> +above. +</P> +<P> +Duplicate names can be useful for patterns where only one instance of the named +capture group can match. Suppose you want to match the name of a weekday, +either as a 3-letter abbreviation or as the full name, and in both cases you +want to extract the abbreviation. This pattern (ignoring the line breaks) does +the job: <pre> + (?J) (?<DN>Mon|Fri|Sun)(?:day)?| (?<DN>Tue)(?:sday)?| (?<DN>Wed)(?:nesday)?| @@ -1921,7 +1953,7 @@ they appear in the overall pattern. The first one that is set is used for the reference. For example, this pattern matches both "foofoo" and "barbar" but not "foobar" or "barfoo": <pre> - (?:(?<n>foo)|(?<n>bar))\k<n> + (?J)(?:(?<n>foo)|(?<n>bar))\k<n> </PRE> </P> @@ -1955,7 +1987,7 @@ items: an escape such as \d or \pL that matches a single character a character class a backreference - a parenthesized group (including most assertions) + a parenthesized group (including lookaround assertions) a subroutine call (recursive or otherwise) </pre> The general repetition quantifier specifies a minimum and maximum number of @@ -2013,8 +2045,10 @@ no characters with a quantifier that has no upper limit, for example: </pre> Earlier versions of Perl and PCRE1 used to give an error at compile time for such patterns. However, because there are cases where this can be useful, such -patterns are now accepted, but if any repetition of the group does in fact -match no characters, the loop is forcibly broken. +patterns are now accepted, but whenever an iteration of such a group matches no +characters, matching moves on to the next item in the pattern instead of +repeatedly matching an empty string. This does not prevent backtracking into +any of the iterations if a subsequent item fails to match. </P> <P> By default, quantifiers are "greedy", that is, they match as much as possible @@ -2341,11 +2375,11 @@ using alternation, as in the example above, or by a quantifier with a minimum of zero. </P> <P> -Backreferences of this type cause the group that they reference to be treated -as an +For versions of PCRE2 less than 10.25, backreferences of this type used to +cause the group that they reference to be treated as an <a href="#atomicgroup">atomic group.</a> -Once the whole group has been matched, a subsequent matching failure cannot -cause backtracking into the middle of the group. +This restriction no longer applies, and backtracking into such groups can occur +as normal. <a name="bigassertions"></a></P> <br><a name="SEC20" href="#TOC1">ASSERTIONS</a><br> <P> @@ -2361,10 +2395,18 @@ those that look behind it, and in each case an assertion may be positive (must match for the assertion to be true) or negative (must not match for the assertion to be true). An assertion group is matched in the normal way, and if it is true, matching continues after it, but with the matching position -in the subject string is was it was before the assertion was processed. +in the subject string reset to what it was before the assertion was processed. +</P> +<P> +The Perl-compatible lookaround assertions are atomic. If an assertion is true, +but there is a subsequent matching failure, there is no backtracking into the +assertion. However, there are some cases where non-atomic assertions can be +useful. PCRE2 has some support for these, described in the section entitled +<a href="#nonatomicassertions">"Non-atomic assertions"</a> +below, but they are not Perl-compatible. </P> <P> -A lookaround assertion may also appear as the condition in a +A lookaround assertion may appear as the condition in a <a href="#conditions">conditional group</a> (see below). In this case, the result of matching the assertion determines which branch of the condition is followed. @@ -2397,36 +2439,23 @@ control passes to the previous backtracking point, thus discarding any captured strings within the assertion. </P> <P> -For compatibility with Perl, most assertion groups may be repeated; though it -makes no sense to assert the same thing several times, the side effect of -capturing may occasionally be useful. However, an assertion that forms the -condition for a conditional group may not be quantified. In practice, for -other assertions, there only three cases: -<br> -<br> -(1) If the quantifier is {0}, the assertion is never obeyed during matching. -However, it may contain internal capture groups that are called from elsewhere -via the -<a href="#groupsassubroutines">subroutine mechanism.</a> -<br> -<br> -(2) If quantifier is {0,n} where n is greater than zero, it is treated as if it -were {0,1}. At run time, the rest of the pattern match is tried with and -without the assertion, the order depending on the greediness of the quantifier. -<br> -<br> -(3) If the minimum repetition is greater than zero, the quantifier is ignored. -The assertion is obeyed just once when encountered during matching. +Most assertion groups may be repeated; though it makes no sense to assert the +same thing several times, the side effect of capturing in positive assertions +may occasionally be useful. However, an assertion that forms the condition for +a conditional group may not be quantified. PCRE2 used to restrict the +repetition of assertions, but from release 10.35 the only restriction is that +an unlimited maximum repetition is changed to be one more than the minimum. For +example, {3,} is treated as {3,4}. </P> <br><b> Alphabetic assertion names </b><br> <P> -Traditionally, symbolic sequences such as (?= and (?<= have been used to specify -lookaround assertions. Perl 5.28 introduced some experimental alphabetic -alternatives which might be easier to remember. They all start with (* instead -of (? and must be written using lower case letters. PCRE2 supports the -following synonyms: +Traditionally, symbolic sequences such as (?= and (?<= have been used to +specify lookaround assertions. Perl 5.28 introduced some experimental +alphabetic alternatives which might be easier to remember. They all start with +(* instead of (? and must be written using lower case letters. PCRE2 supports +the following synonyms: <pre> (*positive_lookahead: or (*pla: is the same as (?= (*negative_lookahead: or (*nla: is the same as (?! @@ -2599,8 +2628,69 @@ preceded by "foo", while </pre> is another pattern that matches "foo" preceded by three digits and any three characters that are not "999". +<a name="nonatomicassertions"></a></P> +<br><a name="SEC21" href="#TOC1">NON-ATOMIC ASSERTIONS</a><br> +<P> +The traditional Perl-compatible lookaround assertions are atomic. That is, if +an assertion is true, but there is a subsequent matching failure, there is no +backtracking into the assertion. However, there are some cases where non-atomic +positive assertions can be useful. PCRE2 provides these using the following +syntax: +<pre> + (*non_atomic_positive_lookahead: or (*napla: or (?* + (*non_atomic_positive_lookbehind: or (*naplb: or (?<* +</pre> +Consider the problem of finding the right-most word in a string that also +appears earlier in the string, that is, it must appear at least twice in total. +This pattern returns the required result as captured substring 1: +<pre> + ^(?x)(*napla: .* \b(\w++)) (?> .*? \b\1\b ){2} +</pre> +For a subject such as "word1 word2 word3 word2 word3 word4" the result is +"word3". How does it work? At the start, ^(?x) anchors the pattern and sets the +"x" option, which causes white space (introduced for readability) to be +ignored. Inside the assertion, the greedy .* at first consumes the entire +string, but then has to backtrack until the rest of the assertion can match a +word, which is captured by group 1. In other words, when the assertion first +succeeds, it captures the right-most word in the string. +</P> +<P> +The current matching point is then reset to the start of the subject, and the +rest of the pattern match checks for two occurrences of the captured word, +using an ungreedy .*? to scan from the left. If this succeeds, we are done, but +if the last word in the string does not occur twice, this part of the pattern +fails. If a traditional atomic lookhead (?= or (*pla: had been used, the +assertion could not be re-entered, and the whole match would fail. The pattern +would succeed only if the very last word in the subject was found twice. +</P> +<P> +Using a non-atomic lookahead, however, means that when the last word does not +occur twice in the string, the lookahead can backtrack and find the second-last +word, and so on, until either the match succeeds, or all words have been +tested. +</P> +<P> +Two conditions must be met for a non-atomic assertion to be useful: the +contents of one or more capturing groups must change after a backtrack into the +assertion, and there must be a backreference to a changed group later in the +pattern. If this is not the case, the rest of the pattern match fails exactly +as before because nothing has changed, so using a non-atomic assertion just +wastes resources. +</P> +<P> +There is one exception to backtracking into a non-atomic assertion. If an +(*ACCEPT) control verb is triggered, the assertion succeeds atomically. That +is, a subsequent match failure cannot backtrack into the assertion. +</P> +<P> +Non-atomic assertions are not supported by the alternative matching function +<b>pcre2_dfa_match()</b>. They are supported by JIT, but only if they do not +contain any control verbs such as (*ACCEPT). (This may change in future). Note +that assertions that appear as conditions for +<a href="#conditions">conditional groups</a> +(see below) must be atomic. </P> -<br><a name="SEC21" href="#TOC1">SCRIPT RUNS</a><br> +<br><a name="SEC22" href="#TOC1">SCRIPT RUNS</a><br> <P> In concept, a script run is a sequence of characters that are all from the same Unicode script such as Latin or Greek. However, because some scripts are @@ -2662,7 +2752,7 @@ parentheses. should not be used within a script run group, because it causes an immediate exit from the group, bypassing the script run checking. <a name="conditions"></a></P> -<br><a name="SEC22" href="#TOC1">CONDITIONAL GROUPS</a><br> +<br><a name="SEC23" href="#TOC1">CONDITIONAL GROUPS</a><br> <P> It is possible to cause the matching process to obey a pattern fragment conditionally or to choose between two alternative fragments, depending on @@ -2838,8 +2928,13 @@ Assertion conditions <P> If the condition is not in any of the above formats, it must be a parenthesized assertion. This may be a positive or negative lookahead or lookbehind -assertion. Consider this pattern, again containing non-significant white space, -and with the two alternatives on the second line: +assertion. However, it must be a traditional atomic assertion, not one of the +PCRE2-specific +<a href="#nonatomicassertions">non-atomic assertions.</a> +</P> +<P> +Consider this pattern, again containing non-significant white space, and with +the two alternatives on the second line: <pre> (?(?=[^a-z]*[a-z]) \d{2}-[a-z]{3}-\d{2} | \d{2}-\d{2}-\d{2} ) @@ -2858,7 +2953,7 @@ positive and negative assertions, because matching always continues after the assertion, whether it succeeds or fails. (Compare non-conditional assertions, for which captures are retained only for positive assertions that succeed.) <a name="comments"></a></P> -<br><a name="SEC23" href="#TOC1">COMMENTS</a><br> +<br><a name="SEC24" href="#TOC1">COMMENTS</a><br> <P> There are two ways of including comments in patterns that are processed by PCRE2. In both cases, the start of the comment must not be in a character @@ -2888,7 +2983,7 @@ a newline in the pattern. The sequence \n is still literal at this stage, so it does not terminate the comment. Only an actual character with the code value 0x0a (the default newline) does so. <a name="recursion"></a></P> -<br><a name="SEC24" href="#TOC1">RECURSIVE PATTERNS</a><br> +<br><a name="SEC25" href="#TOC1">RECURSIVE PATTERNS</a><br> <P> Consider the problem of matching a string in parentheses, allowing for unlimited nested parentheses. Without the use of recursion, the best that can @@ -3076,7 +3171,7 @@ alternative matches "a" and then recurses. In the recursion, \1 does now match "b" and so the whole match succeeds. This match used to fail in Perl, but in later versions (I tried 5.024) it now works. <a name="groupsassubroutines"></a></P> -<br><a name="SEC25" href="#TOC1">GROUPS AS SUBROUTINES</a><br> +<br><a name="SEC26" href="#TOC1">GROUPS AS SUBROUTINES</a><br> <P> If the syntax for a recursive group call (either by number or by name) is used outside the parentheses to which it refers, it operates a bit like a subroutine @@ -3124,7 +3219,7 @@ in groups when called as subroutines is described in the section entitled <a href="#btsub">"Backtracking verbs in subroutines"</a> below. <a name="onigurumasubroutines"></a></P> -<br><a name="SEC26" href="#TOC1">ONIGURUMA SUBROUTINE SYNTAX</a><br> +<br><a name="SEC27" href="#TOC1">ONIGURUMA SUBROUTINE SYNTAX</a><br> <P> For compatibility with Oniguruma, the non-Perl syntax \g followed by a name or a number enclosed either in angle brackets or single quotes, is an alternative @@ -3142,7 +3237,7 @@ plus or a minus sign it is taken as a relative reference. For example: Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are <i>not</i> synonymous. The former is a backreference; the latter is a subroutine call. </P> -<br><a name="SEC27" href="#TOC1">CALLOUTS</a><br> +<br><a name="SEC28" href="#TOC1">CALLOUTS</a><br> <P> Perl has a feature whereby using the sequence (?{...}) causes arbitrary Perl code to be obeyed in the middle of matching a regular expression. This makes it @@ -3218,13 +3313,13 @@ example: </pre> The doubling is removed before the string is passed to the callout function. <a name="backtrackcontrol"></a></P> -<br><a name="SEC28" href="#TOC1">BACKTRACKING CONTROL</a><br> +<br><a name="SEC29" href="#TOC1">BACKTRACKING CONTROL</a><br> <P> There are a number of special "Backtracking Control Verbs" (to use Perl's terminology) that modify the behaviour of backtracking during matching. They are generally of the form (*VERB) or (*VERB:NAME). Some verbs take either form, -possibly behaving differently depending on whether or not a name is present. -The names are not required to be unique within the pattern. +and may behave differently depending on whether or not a name argument is +present. The names are not required to be unique within the pattern. </P> <P> By default, for compatibility with Perl, a name is any sequence of characters @@ -3252,7 +3347,8 @@ PCRE2_ALT_VERBNAMES is also set. The maximum length of a name is 255 in the 8-bit library and 65535 in the 16-bit and 32-bit libraries. If the name is empty, that is, if the closing parenthesis immediately follows the colon, the effect is as if the colon were -not there. Any number of these verbs may occur in a pattern. +not there. Any number of these verbs may occur in a pattern. Except for +(*ACCEPT), they may not be quantified. </P> <P> Since these verbs are specifically related to backtracking, most of them can be @@ -3315,6 +3411,19 @@ This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is captured by the outer parentheses. </P> <P> +(*ACCEPT) is the only backtracking verb that is allowed to be quantified +because an ungreedy quantification with a minimum of zero acts only when a +backtrack happens. Consider, for example, +<pre> + (A(*ACCEPT)??B)C +</pre> +where A, B, and C may be complex expressions. After matching "A", the matcher +processes "BC"; if that fails, causing a backtrack, (*ACCEPT) is triggered and +the match succeeds. In both cases, all but C is captured. Whereas (*COMMIT) +(see below) means "fail on backtrack", a repeated (*ACCEPT) of this type means +"succeed on backtrack". +</P> +<P> <b>Warning:</b> (*ACCEPT) should not be used within a script run group, because it causes an immediate exit from the group, bypassing the script run checking. <pre> @@ -3332,8 +3441,9 @@ A match with the string "aaaa" always fails, but the callout is taken before each backtrack happens (in this example, 10 times). </P> <P> -(*ACCEPT:NAME) and (*FAIL:NAME) are treated as (*MARK:NAME)(*ACCEPT) and -(*MARK:NAME)(*FAIL), respectively. +(*ACCEPT:NAME) and (*FAIL:NAME) behave the same as (*MARK:NAME)(*ACCEPT) and +(*MARK:NAME)(*FAIL), respectively, that is, a (*MARK) is recorded just before +the verb acts. </P> <br><b> Recording which path was taken @@ -3501,6 +3611,12 @@ next attempt at "c". Note that a possessive quantifer does not have the same effect as this example; although it would suppress backtracking during the first match attempt, the second attempt would start at the second character instead of skipping on to "c". +</P> +<P> +If (*SKIP) is used to specify a new starting position that is the same as the +starting position of the current match, or (by being inside a lookbehind) +earlier, the position specified by (*SKIP) is ignored, and instead the normal +"bumpalong" occurs. <pre> (*SKIP:NAME) </pre> @@ -3665,11 +3781,20 @@ retained in both cases. </P> <P> The remaining verbs act only when a later failure causes a backtrack to -reach them. This means that their effect is confined to the assertion, -because lookaround assertions are atomic. A backtrack that occurs after an -assertion is complete does not jump back into the assertion. Note in particular -that a (*MARK) name that is set in an assertion is not "seen" by an instance of -(*SKIP:NAME) latter in the pattern. +reach them. This means that, for the Perl-compatible assertions, their effect +is confined to the assertion, because Perl lookaround assertions are atomic. A +backtrack that occurs after such an assertion is complete does not jump back +into the assertion. Note in particular that a (*MARK) name that is set in an +assertion is not "seen" by an instance of (*SKIP:NAME) later in the pattern. +</P> +<P> +PCRE2 now supports non-atomic positive assertions, as described in the section +entitled +<a href="#nonatomicassertions">"Non-atomic assertions"</a> +above. These assertions must be standalone (not used as conditions). They are +not Perl-compatible. For these assertions, a later backtrack does jump back +into the assertion, and therefore verbs such as (*COMMIT) can be triggered by +backtracks from later in the pattern. </P> <P> The effect of (*THEN) is not allowed to escape beyond an assertion. If there @@ -3711,12 +3836,12 @@ enclosing group that has alternatives (its normal behaviour). However, if there is no such group within the subroutine's group, the subroutine match fails and there is a backtrack at the outer level. </P> -<br><a name="SEC29" href="#TOC1">SEE ALSO</a><br> +<br><a name="SEC30" href="#TOC1">SEE ALSO</a><br> <P> <b>pcre2api</b>(3), <b>pcre2callout</b>(3), <b>pcre2matching</b>(3), <b>pcre2syntax</b>(3), <b>pcre2</b>(3). </P> -<br><a name="SEC30" href="#TOC1">AUTHOR</a><br> +<br><a name="SEC31" href="#TOC1">AUTHOR</a><br> <P> Philip Hazel <br> @@ -3725,11 +3850,11 @@ University Computing Service Cambridge, England. <br> </P> -<br><a name="SEC31" href="#TOC1">REVISION</a><br> +<br><a name="SEC32" href="#TOC1">REVISION</a><br> <P> -Last updated: 12 February 2019 +Last updated: 06 October 2020 <br> -Copyright © 1997-2019 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2syntax.html b/dist2/doc/html/pcre2syntax.html index e3dc1861..7383104a 100644 --- a/dist2/doc/html/pcre2syntax.html +++ b/dist2/doc/html/pcre2syntax.html @@ -32,15 +32,16 @@ please consult the man page, in case the conversion went wrong. <li><a name="TOC17" href="#SEC17">NEWLINE CONVENTION</a> <li><a name="TOC18" href="#SEC18">WHAT \R MATCHES</a> <li><a name="TOC19" href="#SEC19">LOOKAHEAD AND LOOKBEHIND ASSERTIONS</a> -<li><a name="TOC20" href="#SEC20">SCRIPT RUNS</a> -<li><a name="TOC21" href="#SEC21">BACKREFERENCES</a> -<li><a name="TOC22" href="#SEC22">SUBROUTINE REFERENCES (POSSIBLY RECURSIVE)</a> -<li><a name="TOC23" href="#SEC23">CONDITIONAL PATTERNS</a> -<li><a name="TOC24" href="#SEC24">BACKTRACKING CONTROL</a> -<li><a name="TOC25" href="#SEC25">CALLOUTS</a> -<li><a name="TOC26" href="#SEC26">SEE ALSO</a> -<li><a name="TOC27" href="#SEC27">AUTHOR</a> -<li><a name="TOC28" href="#SEC28">REVISION</a> +<li><a name="TOC20" href="#SEC20">NON-ATOMIC LOOKAROUND ASSERTIONS</a> +<li><a name="TOC21" href="#SEC21">SCRIPT RUNS</a> +<li><a name="TOC22" href="#SEC22">BACKREFERENCES</a> +<li><a name="TOC23" href="#SEC23">SUBROUTINE REFERENCES (POSSIBLY RECURSIVE)</a> +<li><a name="TOC24" href="#SEC24">CONDITIONAL PATTERNS</a> +<li><a name="TOC25" href="#SEC25">BACKTRACKING CONTROL</a> +<li><a name="TOC26" href="#SEC26">CALLOUTS</a> +<li><a name="TOC27" href="#SEC27">SEE ALSO</a> +<li><a name="TOC28" href="#SEC28">AUTHOR</a> +<li><a name="TOC29" href="#SEC29">REVISION</a> </ul> <br><a name="SEC1" href="#TOC1">PCRE2 REGULAR EXPRESSION SYNTAX SUMMARY</a><br> <P> @@ -222,6 +223,7 @@ Caucasian_Albanian, Chakma, Cham, Cherokee, +Chorasmian, Common, Coptic, Cuneiform, @@ -229,10 +231,12 @@ Cypriot, Cyrillic, Deseret, Devanagari, +Dives_Akuru, Dogra, Duployan, Egyptian_Hieroglyphs, Elbasan, +Elymaic, Ethiopic, Georgian, Glagolitic, @@ -259,6 +263,7 @@ Kannada, Katakana, Kayah_Li, Kharoshthi, +Khitan_Small_Script, Khmer, Khojki, Khudawadi, @@ -290,10 +295,12 @@ Mro, Multani, Myanmar, Nabataean, +Nandinagari, New_Tai_Lue, Newa, Nko, Nushu, +Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungarian, @@ -344,7 +351,9 @@ Tifinagh, Tirhuta, Ugaritic, Vai, +Wancho, Warang_Citi, +Yezidi, Yi, Zanabazar_Square. </P> @@ -462,7 +471,7 @@ Changes of these options within a group are automatically cancelled at the end of the group. <pre> (?i) caseless - (?J) allow duplicate names + (?J) allow duplicate named groups (?m) multiline (?n) no auto capture (?s) single line (dotall) @@ -544,7 +553,20 @@ setting with a similar syntax. </pre> Each top-level branch of a lookbehind must be of a fixed length. </P> -<br><a name="SEC20" href="#TOC1">SCRIPT RUNS</a><br> +<br><a name="SEC20" href="#TOC1">NON-ATOMIC LOOKAROUND ASSERTIONS</a><br> +<P> +These assertions are specific to PCRE2 and are not Perl-compatible. +<pre> + (?*...) ) + (*napla:...) ) synonyms + (*non_atomic_positive_lookahead:...) ) + + (?<*...) ) + (*naplb:...) ) synonyms + (*non_atomic_positive_lookbehind:...) ) +</PRE> +</P> +<br><a name="SEC21" href="#TOC1">SCRIPT RUNS</a><br> <P> <pre> (*script_run:...) ) script run, can be backtracked into @@ -554,7 +576,7 @@ Each top-level branch of a lookbehind must be of a fixed length. (*asr:...) ) </PRE> </P> -<br><a name="SEC21" href="#TOC1">BACKREFERENCES</a><br> +<br><a name="SEC22" href="#TOC1">BACKREFERENCES</a><br> <P> <pre> \n reference by number (can be ambiguous) @@ -571,7 +593,7 @@ Each top-level branch of a lookbehind must be of a fixed length. (?P=name) reference by name (Python) </PRE> </P> -<br><a name="SEC22" href="#TOC1">SUBROUTINE REFERENCES (POSSIBLY RECURSIVE)</a><br> +<br><a name="SEC23" href="#TOC1">SUBROUTINE REFERENCES (POSSIBLY RECURSIVE)</a><br> <P> <pre> (?R) recurse whole pattern @@ -590,7 +612,7 @@ Each top-level branch of a lookbehind must be of a fixed length. \g'-n' call subroutine by relative number (PCRE2 extension) </PRE> </P> -<br><a name="SEC23" href="#TOC1">CONDITIONAL PATTERNS</a><br> +<br><a name="SEC24" href="#TOC1">CONDITIONAL PATTERNS</a><br> <P> <pre> (?(condition)yes-pattern) @@ -613,7 +635,7 @@ Note the ambiguity of (?(R) and (?(Rn) which might be named reference conditions or recursion tests. Such a condition is interpreted as a reference condition if the relevant named group exists. </P> -<br><a name="SEC24" href="#TOC1">BACKTRACKING CONTROL</a><br> +<br><a name="SEC25" href="#TOC1">BACKTRACKING CONTROL</a><br> <P> All backtracking control verbs may be in the form (*VERB:NAME). For (*MARK) the name is mandatory, for the others it is optional. (*SKIP) changes its behaviour @@ -640,7 +662,7 @@ pattern is not anchored. The effect of one of these verbs in a group called as a subroutine is confined to the subroutine call. </P> -<br><a name="SEC25" href="#TOC1">CALLOUTS</a><br> +<br><a name="SEC26" href="#TOC1">CALLOUTS</a><br> <P> <pre> (?C) callout (assumed number 0) @@ -651,12 +673,12 @@ The allowed string delimiters are ` ' " ^ % # $ (which are the same for the start and the end), and the starting delimiter { matched with the ending delimiter }. To encode the ending delimiter within the string, double it. </P> -<br><a name="SEC26" href="#TOC1">SEE ALSO</a><br> +<br><a name="SEC27" href="#TOC1">SEE ALSO</a><br> <P> <b>pcre2pattern</b>(3), <b>pcre2api</b>(3), <b>pcre2callout</b>(3), <b>pcre2matching</b>(3), <b>pcre2</b>(3). </P> -<br><a name="SEC27" href="#TOC1">AUTHOR</a><br> +<br><a name="SEC28" href="#TOC1">AUTHOR</a><br> <P> Philip Hazel <br> @@ -665,9 +687,9 @@ University Computing Service Cambridge, England. <br> </P> -<br><a name="SEC28" href="#TOC1">REVISION</a><br> +<br><a name="SEC29" href="#TOC1">REVISION</a><br> <P> -Last updated: 11 February 2019 +Last updated: 28 December 2019 <br> Copyright © 1997-2019 University of Cambridge. <br> diff --git a/dist2/doc/html/pcre2test.html b/dist2/doc/html/pcre2test.html index 083d5cc3..4a7e55f7 100644 --- a/dist2/doc/html/pcre2test.html +++ b/dist2/doc/html/pcre2test.html @@ -242,10 +242,17 @@ Behave as if each pattern line has the <b>jit</b> modifier; after successful compilation, each pattern is passed to the just-in-time compiler, if available. </P> <P> +<b>-jitfast</b> +Behave as if each pattern line has the <b>jitfast</b> modifier; after +successful compilation, each pattern is passed to the just-in-time compiler, if +available, and each subject line is passed directly to the JIT matcher via its +"fast path". +</P> +<P> <b>-jitverify</b> Behave as if each pattern line has the <b>jitverify</b> modifier; after successful compilation, each pattern is passed to the just-in-time compiler, if -available, and the use of JIT is verified. +available, and the use of JIT for matching is verified. </P> <P> <b>-LM</b> @@ -254,7 +261,7 @@ standard output, then exit with zero exit code. All other options are ignored. If both -C and -LM are present, whichever is first is recognized. </P> <P> -\fB-pattern\fB <i>modifier-list</i> +<b>-pattern</b> <i>modifier-list</i> Behave as if each pattern line contains the given modifiers. </P> <P> @@ -316,7 +323,7 @@ test data, command lines that begin with # may appear. This file format, with some restrictions, can also be processed by the <b>perltest.sh</b> script that is distributed with PCRE2 as a means of checking that the behaviour of PCRE2 and Perl is the same. For a specification of <b>perltest.sh</b>, see the -comments near its beginning. +comments near its beginning. See also the #perltest command below. </P> <P> When the input is a terminal, <b>pcre2test</b> prompts for each line of input, @@ -369,6 +376,12 @@ This command is used to load a set of precompiled patterns from a file, as described in the section entitled "Saving and restoring compiled patterns" <a href="#saverestore">below.</a> <pre> + #loadtables <filename> +</pre> +This command is used to load a set of binary character tables that can be +accessed by the tables=3 qualifier. Such tables can be created by the +<b>pcre2_dftables</b> program with the -b option. +<pre> #newline_default [<newline-list>] </pre> When PCRE2 is built, a default newline convention can be specified. This @@ -407,14 +420,20 @@ patterns. Modifiers on a pattern can change these settings. <pre> #perltest </pre> -The appearance of this line causes all subsequent modifier settings to be -checked for compatibility with the <b>perltest.sh</b> script, which is used to -confirm that Perl gives the same results as PCRE2. Also, apart from comment -lines, #pattern commands, and #subject commands that set or unset "mark", no -command lines are permitted, because they and many of the modifiers are -specific to <b>pcre2test</b>, and should not be used in test files that are also -processed by <b>perltest.sh</b>. The <b>#perltest</b> command helps detect tests -that are accidentally put in the wrong file. +This line is used in test files that can also be processed by <b>perltest.sh</b> +to confirm that Perl gives the same results as PCRE2. Subsequent tests are +checked for the use of <b>pcre2test</b> features that are incompatible with the +<b>perltest.sh</b> script. +</P> +<P> +Patterns must use '/' as their delimiter, and only certain modifiers are +supported. Comment lines, #pattern commands, and #subject commands that set or +unset "mark" are recognized and acted on. The #perltest, #forbid_utf, and +#newline_default commands, which are needed in the relevant pcre2test files, +are silently ignored. All other command lines are ignored, but give a warning +message. The <b>#perltest</b> command helps detect tests that are accidentally +put in the wrong file or use the wrong delimiter. For more details of the +<b>perltest.sh</b> script see the comments it contains. <pre> #pop [<modifiers>] #popcopy [<modifiers>] @@ -613,6 +632,7 @@ for a description of the effects of these options. firstline set PCRE2_FIRSTLINE literal set PCRE2_LITERAL match_line set PCRE2_EXTRA_MATCH_LINE + match_invalid_utf set PCRE2_MATCH_INVALID_UTF match_unset_backref set PCRE2_MATCH_UNSET_BACKREF match_word set PCRE2_EXTRA_MATCH_WORD /m multiline set PCRE2_MULTILINE @@ -671,7 +691,7 @@ heavily used in the test files. pushcopy push a copy onto the stack stackguard=<number> test the stackguard feature subject_literal treat all subject lines as literal - tables=[0|1|2] select internal tables + tables=[0|1|2|3] select internal tables use_length do not zero-terminate the pattern utf8_input treat input as UTF-8 </pre> @@ -738,7 +758,9 @@ options, the line is omitted. "First code unit" is where any match must start; if there is more than one they are listed as "starting code units". "Last code unit" is the last literal code unit that must be present in any match. This is not necessarily the last character. These lines are omitted if no starting or -ending code units are recorded. +ending code units are recorded. The subject length line is omitted when +<b>no_start_optimize</b> is set because the minimum length is not calculated +when it can never be used. </P> <P> The <b>framesize</b> modifier shows the size, in bytes, of the storage frames @@ -1017,18 +1039,20 @@ Using alternative character tables </b><br> <P> The value specified for the <b>tables</b> modifier must be one of the digits 0, -1, or 2. It causes a specific set of built-in character tables to be passed to -<b>pcre2_compile()</b>. This is used in the PCRE2 tests to check behaviour with -different character tables. The digit specifies the tables as follows: +1, 2, or 3. It causes a specific set of built-in character tables to be passed +to <b>pcre2_compile()</b>. This is used in the PCRE2 tests to check behaviour +with different character tables. The digit specifies the tables as follows: <pre> 0 do not pass any special character tables 1 the default ASCII tables, as distributed in pcre2_chartables.c.dist 2 a set of tables defining ISO 8859 characters + 3 a set of tables loaded by the #loadtables command </pre> -In table 2, some characters whose codes are greater than 128 are identified as -letters, digits, spaces, etc. Setting alternate character tables and a locale -are mutually exclusive. +In tables 2, some characters whose codes are greater than 128 are identified as +letters, digits, spaces, etc. Tables 3 can be used only after a +<b>#loadtables</b> command has loaded them from a binary file. Setting alternate +character tables and a locale are mutually exclusive. </P> <br><b> Setting certain match controls @@ -1040,24 +1064,27 @@ modifier list, in which case they are applied to every subject line that is processed with that pattern. These modifiers do not affect the compilation process. <pre> - aftertext show text after match - allaftertext show text after captures - allcaptures show all captures - allvector show the entire ovector - allusedtext show all consulted text - altglobal alternative global matching - /g global global matching - jitstack=<n> set size of JIT stack - mark show mark values - replace=<string> specify a replacement string - startchar show starting character when relevant - substitute_callout use substitution callouts - substitute_extended use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n - substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH - substitute_stop=<n> skip substitution number n and greater - substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET - substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY + aftertext show text after match + allaftertext show text after captures + allcaptures show all captures + allvector show the entire ovector + allusedtext show all consulted text + altglobal alternative global matching + /g global global matching + jitstack=<n> set size of JIT stack + mark show mark values + replace=<string> specify a replacement string + startchar show starting character when relevant + substitute_callout use substitution callouts + substitute_extended use PCRE2_SUBSTITUTE_EXTENDED + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED + substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution <n> + substitute_stop=<n> skip substitution <n> and following + substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET + substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY </pre> These modifiers may not appear in a <b>#pattern</b> command. If you want them as defaults, set them in a <b>#subject</b> command. @@ -1223,8 +1250,11 @@ pattern. startoffset=<n> same as offset=<n> substitute_callout use substitution callouts substitute_extedded use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution number n substitute_stop=<n> skip substitution number n and greater substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY @@ -1249,22 +1279,27 @@ following line with a plus character following the capture number. </P> <P> The <b>allusedtext</b> modifier requests that all the text that was consulted -during a successful pattern match by the interpreter should be shown. This -feature is not supported for JIT matching, and if requested with JIT it is -ignored (with a warning message). Setting this modifier affects the output if -there is a lookbehind at the start of a match, or a lookahead at the end, or if -\K is used in the pattern. Characters that precede or follow the start and end -of the actual match are indicated in the output by '<' or '>' characters -underneath them. Here is an example: +during a successful pattern match by the interpreter should be shown, for both +full and partial matches. This feature is not supported for JIT matching, and +if requested with JIT it is ignored (with a warning message). Setting this +modifier affects the output if there is a lookbehind at the start of a match, +or, for a complete match, a lookahead at the end, or if \K is used in the +pattern. Characters that precede or follow the start and end of the actual +match are indicated in the output by '<' or '>' characters underneath them. +Here is an example: <pre> re> /(?<=pqr)abc(?=xyz)/ data> 123pqrabcxyz456\=allusedtext 0: pqrabcxyz <<< >>> + data> 123pqrabcxy\=ph,allusedtext + Partial match: pqrabcxy + <<< </pre> -This shows that the matched string is "abc", with the preceding and following -strings "pqr" and "xyz" having been consulted during the match (when processing -the assertions). +The first, complete match shows that the matched string is "abc", with the +preceding and following strings "pqr" and "xyz" having been consulted during +the match (when processing the assertions). The partial match can indicate only +the preceding string. </P> <P> The <b>startchar</b> modifier requests that the starting character for the match @@ -1380,9 +1415,10 @@ Testing the substitution function </b><br> <P> If the <b>replace</b> modifier is set, the <b>pcre2_substitute()</b> function is -called instead of one of the matching functions. Note that replacement strings -cannot contain commas, because a comma signifies the end of a modifier. This is -not thought to be an issue in a test program. +called instead of one of the matching functions (or after one call of +<b>pcre2_match()</b> in the case of PCRE2_SUBSTITUTE_MATCHED). Note that +replacement strings cannot contain commas, because a comma signifies the end of +a modifier. This is not thought to be an issue in a test program. </P> <P> Unlike subject strings, <b>pcre2test</b> does not process replacement strings @@ -1398,11 +1434,16 @@ for <b>pcre2_substitute()</b>: <pre> global PCRE2_SUBSTITUTE_GLOBAL substitute_extended PCRE2_SUBSTITUTE_EXTENDED + substitute_literal PCRE2_SUBSTITUTE_LITERAL + substitute_matched PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only PCRE2_SUBSTITUTE_REPLACEMENT_ONLY substitute_unknown_unset PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty PCRE2_SUBSTITUTE_UNSET_EMPTY - -</PRE> +</pre> +See the +<a href="pcre2api.html"><b>pcre2api</b></a> +documentation for details of these options. </P> <P> After a successful substitution, the modified string is output, preceded by the @@ -1779,7 +1820,7 @@ restart the match with additional subject data by means of the <b>dfa_restart</b> modifier. For example: <pre> re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ - data> 23ja\=P,dfa + data> 23ja\=ps,dfa Partial match: 23ja data> n05\=dfa,dfa_restart 0: n05 @@ -2078,9 +2119,9 @@ Cambridge, England. </P> <br><a name="SEC21" href="#TOC1">REVISION</a><br> <P> -Last updated: 11 March 2019 +Last updated: 14 September 2020 <br> -Copyright © 1997-2019 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/html/pcre2unicode.html b/dist2/doc/html/pcre2unicode.html index c11c7c2b..76ca6ea2 100644 --- a/dist2/doc/html/pcre2unicode.html +++ b/dist2/doc/html/pcre2unicode.html @@ -16,22 +16,33 @@ please consult the man page, in case the conversion went wrong. UNICODE AND UTF SUPPORT </b><br> <P> -When PCRE2 is built with Unicode support (which is the default), it has -knowledge of Unicode character properties and can process text strings in -UTF-8, UTF-16, or UTF-32 format (depending on the code unit width). However, by -default, PCRE2 assumes that one code unit is one character. To process a -pattern as a UTF string, where a character may require more than one code unit, -you must call +PCRE2 is normally built with Unicode support, though if you do not need it, you +can build it without, in which case the library will be smaller. With Unicode +support, PCRE2 has knowledge of Unicode character properties and can process +strings of text in UTF-8, UTF-16, and UTF-32 format (depending on the code unit +width), but this is not the default. Unless specifically requested, PCRE2 +treats each code unit in a string as one character. +</P> +<P> +There are two ways of telling PCRE2 to switch to UTF mode, where characters may +consist of more than one code unit and the range of values is constrained. The +program can call <a href="pcre2_compile.html"><b>pcre2_compile()</b></a> -with the PCRE2_UTF option flag, or the pattern must start with the sequence -(*UTF). When either of these is the case, both the pattern and any subject -strings that are matched against it are treated as UTF strings instead of -strings of individual one-code-unit characters. There are also some other -changes to the way characters are handled, as documented below. +with the PCRE2_UTF option, or the pattern may start with the sequence (*UTF). +However, the latter facility can be locked out by the PCRE2_NEVER_UTF option. +That is, the programmer can prevent the supplier of the pattern from switching +to UTF mode. +</P> +<P> +Note that the PCRE2_MATCH_INVALID_UTF option (see +<a href="#matchinvalid">below)</a> +forces PCRE2_UTF to be set. </P> <P> -If you do not need Unicode support you can build PCRE2 without it, in which -case the library will be smaller. +In UTF mode, both the pattern and any subject strings that are matched against +it are treated as UTF strings instead of strings of individual one-code-unit +characters. There are also some other changes to the way characters are +handled, as documented below. </P> <br><b> UNICODE PROPERTY SUPPORT @@ -63,22 +74,22 @@ also recognized; larger ones can be coded using \o{...}. <P> The escape sequence \N{U+<hex digits>} is recognized as another way of specifying a Unicode character by code point in a UTF mode. It is not allowed -in non-UTF modes. +in non-UTF mode. </P> <P> -In UTF modes, repeat quantifiers apply to complete UTF characters, not to +In UTF mode, repeat quantifiers apply to complete UTF characters, not to individual code units. </P> <P> -In UTF modes, the dot metacharacter matches one UTF character instead of a +In UTF mode, the dot metacharacter matches one UTF character instead of a single code unit. </P> <P> -In UTF modes, capture group names are not restricted to ASCII, and may contain +In UTF mode, capture group names are not restricted to ASCII, and may contain any Unicode letters and decimal digits, as well as underscore. </P> <P> -The escape sequence \C can be used to match a single code unit in a UTF mode, +The escape sequence \C can be used to match a single code unit in UTF mode, but its use can lead to some strange effects because it breaks up multi-unit characters (see the description of \C in the <a href="pcre2pattern.html"><b>pcre2pattern</b></a> @@ -93,7 +104,7 @@ may consist of more than one code unit. The use of \C in these modes provokes a match-time error. Also, the JIT optimization does not support \C in these modes. If JIT optimization is requested for a UTF-8 or UTF-16 pattern that contains \C, it will not succeed, and so when <b>pcre2_match()</b> is called, -the matching will be carried out by the normal interpretive function. +the matching will be carried out by the interpretive function. </P> <P> The character escapes \b, \B, \d, \D, \s, \S, \w, and \W correctly test @@ -123,14 +134,16 @@ However, the special horizontal and vertical white space matching escapes (\h, not PCRE2_UCP is set. </P> <br><b> -CASE-EQUIVALENCE IN UTF MODES +UNICODE CASE-EQUIVALENCE </b><br> <P> -Case-insensitive matching in a UTF mode makes use of Unicode properties except -for characters whose code points are less than 128 and that have at most two -case-equivalent values. For these, a direct table lookup is used for speed. A -few Unicode characters such as Greek sigma have more than two code points that -are case-equivalent, and these are treated as such. +If either PCRE2_UTF or PCRE2_UCP is set, upper/lower case processing makes use +of Unicode properties except for characters whose code points are less than 128 +and that have at most two case-equivalent values. For these, a direct table +lookup is used for speed. A few Unicode characters such as Greek sigma have +more than two code points that are case-equivalent, and these are treated +specially. Setting PCRE2_UCP without PCRE2_UTF allows Unicode-style case +processing for non-UTF character encodings such as UCS-2. <a name="scriptruns"></a></P> <br><b> SCRIPT RUNS @@ -248,7 +261,7 @@ VALIDITY OF UTF STRINGS <P> When the PCRE2_UTF option is set, the strings passed as patterns and subjects are (by default) checked for validity on entry to the relevant functions. If an -invalid UTF string is passed, an negative error code is returned. The code unit +invalid UTF string is passed, a negative error code is returned. The code unit offset to the offending character can be extracted from the match data block by calling <b>pcre2_get_startchar()</b>, which is used for this purpose after a UTF error. @@ -263,17 +276,16 @@ only valid UTF code unit sequences. </P> <P> If you pass an invalid UTF string when PCRE2_NO_UTF_CHECK is set, the result -is usually undefined and your program may crash or loop indefinitely. There is, -however, one mode of matching that can handle invalid UTF subject strings. This -is matching via the JIT optimization using the PCRE2_JIT_INVALID_UTF option -when calling <b>pcre2_jit_compile()</b>. For details, see the -<a href="pcre2jit.html"><b>pcre2jit</b></a> -documentation. +is undefined and your program may crash or loop indefinitely or give incorrect +results. There is, however, one mode of matching that can handle invalid UTF +subject strings. This is enabled by passing PCRE2_MATCH_INVALID_UTF to +<b>pcre2_compile()</b> and is discussed below in the next section. The rest of +this section covers the case when PCRE2_MATCH_INVALID_UTF is not set. </P> <P> -Passing PCRE2_NO_UTF_CHECK to <b>pcre2_compile()</b> just disables the check for -the pattern; it does not also apply to subject strings. If you want to disable -the check for a subject string you must pass this same option to +Passing PCRE2_NO_UTF_CHECK to <b>pcre2_compile()</b> just disables the UTF check +for the pattern; it does not also apply to subject strings. If you want to +disable the check for a subject string you must pass this same option to <b>pcre2_match()</b> or <b>pcre2_dfa_match()</b>. </P> <P> @@ -352,7 +364,7 @@ these code points are excluded by RFC 3629. <pre> PCRE2_ERROR_UTF8_ERR13 </pre> -A 4-byte character has a value greater than 0x10fff; these code points are +A 4-byte character has a value greater than 0x10ffff; these code points are excluded by RFC 3629. <pre> PCRE2_ERROR_UTF8_ERR14 @@ -405,7 +417,59 @@ The following negative error codes are given for invalid UTF-32 strings: PCRE2_ERROR_UTF32_ERR1 Surrogate character (0xd800 to 0xdfff) PCRE2_ERROR_UTF32_ERR2 Code point is greater than 0x10ffff -</PRE> +<a name="matchinvalid"></a></PRE> +</P> +<br><b> +MATCHING IN INVALID UTF STRINGS +</b><br> +<P> +You can run pattern matches on subject strings that may contain invalid UTF +sequences if you call <b>pcre2_compile()</b> with the PCRE2_MATCH_INVALID_UTF +option. This is supported by <b>pcre2_match()</b>, including JIT matching, but +not by <b>pcre2_dfa_match()</b>. When PCRE2_MATCH_INVALID_UTF is set, it forces +PCRE2_UTF to be set as well. Note, however, that the pattern itself must be a +valid UTF string. +</P> +<P> +Setting PCRE2_MATCH_INVALID_UTF does not affect what <b>pcre2_compile()</b> +generates, but if <b>pcre2_jit_compile()</b> is subsequently called, it does +generate different code. If JIT is not used, the option affects the behaviour +of the interpretive code in <b>pcre2_match()</b>. When PCRE2_MATCH_INVALID_UTF +is set at compile time, PCRE2_NO_UTF_CHECK is ignored at match time. +</P> +<P> +In this mode, an invalid code unit sequence in the subject never matches any +pattern item. It does not match dot, it does not match \p{Any}, it does not +even match negative items such as [^X]. A lookbehind assertion fails if it +encounters an invalid sequence while moving the current point backwards. In +other words, an invalid UTF code unit sequence acts as a barrier which no match +can cross. +</P> +<P> +You can also think of this as the subject being split up into fragments of +valid UTF, delimited internally by invalid code unit sequences. The pattern is +matched fragment by fragment. The result of a successful match, however, is +given as code unit offsets in the entire subject string in the usual way. There +are a few points to consider: +</P> +<P> +The internal boundaries are not interpreted as the beginnings or ends of lines +and so do not match circumflex or dollar characters in the pattern. +</P> +<P> +If <b>pcre2_match()</b> is called with an offset that points to an invalid +UTF-sequence, that sequence is skipped, and the match starts at the next valid +UTF character, or the end of the subject. +</P> +<P> +At internal fragment boundaries, \b and \B behave in the same way as at the +beginning and end of the subject. For example, a sequence such as \bWORD\b +would match an instance of WORD that is surrounded by invalid UTF code units. +</P> +<P> +Using PCRE2_MATCH_INVALID_UTF, an application can run matches on arbitrary +data, knowing that any matched strings that are returned are valid UTF. This +can be useful when searching for UTF text in executable or other binary files. </P> <br><b> AUTHOR @@ -422,9 +486,9 @@ Cambridge, England. REVISION </b><br> <P> -Last updated: 06 March 2019 +Last updated: 23 February 2020 <br> -Copyright © 1997-2019 University of Cambridge. +Copyright © 1997-2020 University of Cambridge. <br> <p> Return to the <a href="index.html">PCRE2 index page</a>. diff --git a/dist2/doc/index.html.src b/dist2/doc/index.html.src index 82b43c35..2c7c5fb2 100644 --- a/dist2/doc/index.html.src +++ b/dist2/doc/index.html.src @@ -146,6 +146,9 @@ in the library. <tr><td><a href="pcre2_get_mark.html">pcre2_get_mark</a></td> <td> Get a (*MARK) name</td></tr> +<tr><td><a href="pcre2_get_match_data_size.html">pcre2_get_match_data_size</a></td> + <td> Get the size of a match data block</td></tr> + <tr><td><a href="pcre2_get_ovector_count.html">pcre2_get_ovector_count</a></td> <td> Get the ovector count</td></tr> @@ -176,6 +179,9 @@ in the library. <tr><td><a href="pcre2_maketables.html">pcre2_maketables</a></td> <td> Build character tables in current locale</td></tr> +<tr><td><a href="pcre2_maketables_free.html">pcre2_maketables_free</a></td> + <td> Free character tables</td></tr> + <tr><td><a href="pcre2_match.html">pcre2_match</a></td> <td> Match a compiled pattern to a subject string (Perl compatible)</td></tr> diff --git a/dist2/doc/pcre2-config.txt b/dist2/doc/pcre2-config.txt index 8ddea2a3..33785f4b 100644 --- a/dist2/doc/pcre2-config.txt +++ b/dist2/doc/pcre2-config.txt @@ -16,8 +16,8 @@ DESCRIPTION pcre2-config returns the configuration of the installed PCRE2 libraries and the options required to compile a program to use them. Some of the - options apply only to the 8-bit, or 16-bit, or 32-bit libraries, - respectively, and are not available for libraries that have not been + options apply only to the 8-bit, or 16-bit, or 32-bit libraries, re- + spectively, and are not available for libraries that have not been built. If an unavailable option is encountered, the "usage" information is output. @@ -36,30 +36,30 @@ OPTIONS --version Writes the version number of the installed PCRE2 libraries to the standard output. - --libs8 Writes to the standard output the command line options - required to link with the 8-bit PCRE2 library (-lpcre2-8 on + --libs8 Writes to the standard output the command line options re- + quired to link with the 8-bit PCRE2 library (-lpcre2-8 on many systems). - --libs16 Writes to the standard output the command line options - required to link with the 16-bit PCRE2 library (-lpcre2-16 on + --libs16 Writes to the standard output the command line options re- + quired to link with the 16-bit PCRE2 library (-lpcre2-16 on many systems). - --libs32 Writes to the standard output the command line options - required to link with the 32-bit PCRE2 library (-lpcre2-32 on + --libs32 Writes to the standard output the command line options re- + quired to link with the 32-bit PCRE2 library (-lpcre2-32 on many systems). --libs-posix - Writes to the standard output the command line options - required to link with PCRE2's POSIX API wrapper library + Writes to the standard output the command line options re- + quired to link with PCRE2's POSIX API wrapper library (-lpcre2-posix -lpcre2-8 on many systems). - --cflags Writes to the standard output the command line options - required to compile files that use PCRE2 (this may include - some -I options, but is blank on many systems). + --cflags Writes to the standard output the command line options re- + quired to compile files that use PCRE2 (this may include some + -I options, but is blank on many systems). --cflags-posix - Writes to the standard output the command line options - required to compile files that use PCRE2's POSIX API wrapper + Writes to the standard output the command line options re- + quired to compile files that use PCRE2's POSIX API wrapper library (this may include some -I options, but is blank on many systems). diff --git a/dist2/doc/pcre2.txt b/dist2/doc/pcre2.txt index 9c956e8c..365ecc49 100644 --- a/dist2/doc/pcre2.txt +++ b/dist2/doc/pcre2.txt @@ -31,8 +31,8 @@ INTRODUCTION appeared in Python and the original PCRE before they appeared in Perl are available using the Python syntax. There is also some support for one or two .NET and Oniguruma syntax items, and there are options for - requesting some minor changes that give better ECMAScript (aka - JavaScript) compatibility. + requesting some minor changes that give better ECMAScript (aka Java- + Script) compatibility. The source code for PCRE2 can be compiled to support 8-bit, 16-bit, or 32-bit code units, which means that up to three separate libraries may @@ -111,9 +111,9 @@ SECURITY CONSIDERATIONS The use of the \C escape sequence in a UTF-8 or UTF-16 pattern can lead to problems, because it may leave the current matching point in the - middle of a multi-code-unit character. The PCRE2_NEVER_BACKSLASH_C - option can be used by an application to lock out the use of \C, causing - a compile-time error if it is encountered. It is also possible to build + middle of a multi-code-unit character. The PCRE2_NEVER_BACKSLASH_C op- + tion can be used by an application to lock out the use of \C, causing a + compile-time error if it is encountered. It is also possible to build PCRE2 with the use of \C permanently disabled. Another way that performance can be hit is by running a pattern that @@ -261,7 +261,7 @@ PCRE2 NATIVE API COMPILE CONTEXT FUNCTIONS uint32_t value); int pcre2_set_character_tables(pcre2_compile_context *ccontext, - const unsigned char *tables); + const uint8_t *tables); int pcre2_set_compile_extra_options(pcre2_compile_context *ccontext, uint32_t extra_options); @@ -351,7 +351,7 @@ PCRE2 NATIVE API STRING SUBSTITUTION FUNCTION int pcre2_substitute(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, PCRE2_SIZE startoffset, uint32_t options, pcre2_match_data *match_data, - pcre2_match_context *mcontext, PCRE2_SPTR replacementzfP, + pcre2_match_context *mcontext, PCRE2_SPTR replacementz, PCRE2_SIZE rlength, PCRE2_UCHAR *outputbuffer, PCRE2_SIZE *outlengthptr); @@ -400,7 +400,10 @@ PCRE2 NATIVE API AUXILIARY FUNCTIONS int pcre2_get_error_message(int errorcode, PCRE2_UCHAR *buffer, PCRE2_SIZE bufflen); - const unsigned char *pcre2_maketables(pcre2_general_context *gcontext); + const uint8_t *pcre2_maketables(pcre2_general_context *gcontext); + + void pcre2_maketables_free(pcre2_general_context *gcontext, + const uint8_t *tables); int pcre2_pattern_info(const pcre2_code *code, uint32_t what, void *where); @@ -451,8 +454,8 @@ PCRE2 EXPERIMENTAL PATTERN CONVERSION FUNCTIONS void pcre2_converted_pattern_free(PCRE2_UCHAR *converted_pattern); These functions provide a way of converting non-PCRE2 patterns into - patterns that can be processed by pcre2_compile(). This facility is - experimental and may be changed in future releases. At present, "globs" + patterns that can be processed by pcre2_compile(). This facility is ex- + perimental and may be changed in future releases. At present, "globs" and POSIX basic and extended patterns can be converted. Details are given in the pcre2convert documentation. @@ -543,22 +546,22 @@ PCRE2 API OVERVIEW The compiling and matching functions recognize various options that are passed as bits in an options argument. There are also some more compli- - cated parameters such as custom memory management functions and - resource limits that are passed in "contexts" (which are just memory + cated parameters such as custom memory management functions and re- + source limits that are passed in "contexts" (which are just memory blocks, described below). Simple applications do not need to make use of contexts. Just-in-time (JIT) compiler support is an optional feature of PCRE2 that can be built in appropriate hardware environments. It greatly - speeds up the matching performance of many patterns. Programs can - request that it be used if available by calling pcre2_jit_compile() - after a pattern has been successfully compiled by pcre2_compile(). This - does nothing if JIT support is not available. + speeds up the matching performance of many patterns. Programs can re- + quest that it be used if available by calling pcre2_jit_compile() after + a pattern has been successfully compiled by pcre2_compile(). This does + nothing if JIT support is not available. More complicated programs might need to make use of the specialist functions pcre2_jit_stack_create(), pcre2_jit_stack_free(), and - pcre2_jit_stack_assign() in order to control the JIT code's memory - usage. + pcre2_jit_stack_assign() in order to control the JIT code's memory us- + age. JIT matching is automatically used by pcre2_match() if it is available, unless the PCRE2_NO_JIT option is set. There is also a direct interface @@ -570,13 +573,12 @@ PCRE2 API OVERVIEW patible, is also provided. This uses a different algorithm for the matching. The alternative algorithm finds all possible matches (at a given point in the subject), and scans the subject just once (unless - there are lookaround assertions). However, this algorithm does not - return captured substrings. A description of the two matching algo- - rithms and their advantages and disadvantages is given in the - pcre2matching documentation. There is no JIT support for - pcre2_dfa_match(). + there are lookaround assertions). However, this algorithm does not re- + turn captured substrings. A description of the two matching algorithms + and their advantages and disadvantages is given in the pcre2matching + documentation. There is no JIT support for pcre2_dfa_match(). - In addition to the main compiling and matching functions, there are + In addition to the main compiling and matching functions, there are convenience functions for extracting captured substrings from a subject string that has been matched by pcre2_match(). They are: @@ -590,105 +592,106 @@ PCRE2 API OVERVIEW pcre2_substring_nametable_scan() pcre2_substring_number_from_name() - pcre2_substring_free() and pcre2_substring_list_free() are also pro- - vided, to free memory used for extracted strings. If either of these - functions is called with a NULL argument, the function returns immedi- + pcre2_substring_free() and pcre2_substring_list_free() are also pro- + vided, to free memory used for extracted strings. If either of these + functions is called with a NULL argument, the function returns immedi- ately without doing anything. - The function pcre2_substitute() can be called to match a pattern and - return a copy of the subject string with substitutions for parts that + The function pcre2_substitute() can be called to match a pattern and + return a copy of the subject string with substitutions for parts that were matched. - Functions whose names begin with pcre2_serialize_ are used for saving + Functions whose names begin with pcre2_serialize_ are used for saving compiled patterns on disc or elsewhere, and reloading them later. - Finally, there are functions for finding out information about a com- - piled pattern (pcre2_pattern_info()) and about the configuration with + Finally, there are functions for finding out information about a com- + piled pattern (pcre2_pattern_info()) and about the configuration with which PCRE2 was built (pcre2_config()). - Functions with names ending with _free() are used for freeing memory - blocks of various sorts. In all cases, if one of these functions is + Functions with names ending with _free() are used for freeing memory + blocks of various sorts. In all cases, if one of these functions is called with a NULL argument, it does nothing. STRING LENGTHS AND OFFSETS - The PCRE2 API uses string lengths and offsets into strings of code - units in several places. These values are always of type PCRE2_SIZE, - which is an unsigned integer type, currently always defined as size_t. - The largest value that can be stored in such a type (that is - ~(PCRE2_SIZE)0) is reserved as a special indicator for zero-terminated - strings and unset offsets. Therefore, the longest string that can be + The PCRE2 API uses string lengths and offsets into strings of code + units in several places. These values are always of type PCRE2_SIZE, + which is an unsigned integer type, currently always defined as size_t. + The largest value that can be stored in such a type (that is + ~(PCRE2_SIZE)0) is reserved as a special indicator for zero-terminated + strings and unset offsets. Therefore, the longest string that can be handled is one less than this maximum. NEWLINES PCRE2 supports five different conventions for indicating line breaks in - strings: a single CR (carriage return) character, a single LF (line- + strings: a single CR (carriage return) character, a single LF (line- feed) character, the two-character sequence CRLF, any of the three pre- - ceding, or any Unicode newline sequence. The Unicode newline sequences - are the three just mentioned, plus the single characters VT (vertical + ceding, or any Unicode newline sequence. The Unicode newline sequences + are the three just mentioned, plus the single characters VT (vertical tab, U+000B), FF (form feed, U+000C), NEL (next line, U+0085), LS (line separator, U+2028), and PS (paragraph separator, U+2029). - Each of the first three conventions is used by at least one operating + Each of the first three conventions is used by at least one operating system as its standard newline sequence. When PCRE2 is built, a default can be specified. If it is not, the default is set to LF, which is the - Unix standard. However, the newline convention can be changed by an - application when calling pcre2_compile(), or it can be specified by - special text at the start of the pattern itself; this overrides any - other settings. See the pcre2pattern page for details of the special - character sequences. + Unix standard. However, the newline convention can be changed by an ap- + plication when calling pcre2_compile(), or it can be specified by spe- + cial text at the start of the pattern itself; this overrides any other + settings. See the pcre2pattern page for details of the special charac- + ter sequences. - In the PCRE2 documentation the word "newline" is used to mean "the + In the PCRE2 documentation the word "newline" is used to mean "the character or pair of characters that indicate a line break". The choice - of newline convention affects the handling of the dot, circumflex, and + of newline convention affects the handling of the dot, circumflex, and dollar metacharacters, the handling of #-comments in /x mode, and, when - CRLF is a recognized line ending sequence, the match position advance- + CRLF is a recognized line ending sequence, the match position advance- ment for a non-anchored pattern. There is more detail about this in the section on pcre2_match() options below. - The choice of newline convention does not affect the interpretation of + The choice of newline convention does not affect the interpretation of the \n or \r escape sequences, nor does it affect what \R matches; this has its own separate convention. MULTITHREADING - In a multithreaded application it is important to keep thread-specific - data separate from data that can be shared between threads. The PCRE2 - library code itself is thread-safe: it contains no static or global - variables. The API is designed to be fairly simple for non-threaded - applications while at the same time ensuring that multithreaded appli- - cations can use it. + In a multithreaded application it is important to keep thread-specific + data separate from data that can be shared between threads. The PCRE2 + library code itself is thread-safe: it contains no static or global + variables. The API is designed to be fairly simple for non-threaded ap- + plications while at the same time ensuring that multithreaded applica- + tions can use it. There are several different blocks of data that are used to pass infor- mation between the application and the PCRE2 libraries. The compiled pattern - A pointer to the compiled form of a pattern is returned to the user + A pointer to the compiled form of a pattern is returned to the user when pcre2_compile() is successful. The data in the compiled pattern is - fixed, and does not change when the pattern is matched. Therefore, it - is thread-safe, that is, the same compiled pattern can be used by more + fixed, and does not change when the pattern is matched. Therefore, it + is thread-safe, that is, the same compiled pattern can be used by more than one thread simultaneously. For example, an application can compile all its patterns at the start, before forking off multiple threads that - use them. However, if the just-in-time (JIT) optimization feature is - being used, it needs separate memory stack areas for each thread. See + use them. However, if the just-in-time (JIT) optimization feature is + being used, it needs separate memory stack areas for each thread. See the pcre2jit documentation for more details. - In a more complicated situation, where patterns are compiled only when - they are first needed, but are still shared between threads, pointers - to compiled patterns must be protected from simultaneous writing by - multiple threads, at least until a pattern has been compiled. The logic - can be something like this: + In a more complicated situation, where patterns are compiled only when + they are first needed, but are still shared between threads, pointers + to compiled patterns must be protected from simultaneous writing by + multiple threads. This is somewhat tricky to do correctly. If you know + that writing to a pointer is atomic in your environment, you can use + logic like this: Get a read-only (shared) lock (mutex) for pointer if (pointer == NULL) { Get a write (unique) lock for pointer - pointer = pcre2_compile(... + if (pointer == NULL) pointer = pcre2_compile(... } Release the lock Use pointer in pcre2_match() @@ -696,13 +699,42 @@ MULTITHREADING Of course, testing for compilation errors should also be included in the code. + The reason for checking the pointer a second time is as follows: Sev- + eral threads may have acquired the shared lock and tested the pointer + for being NULL, but only one of them will be given the write lock, with + the rest kept waiting. The winning thread will compile the pattern and + store the result. After this thread releases the write lock, another + thread will get it, and if it does not retest pointer for being NULL, + will recompile the pattern and overwrite the pointer, creating a memory + leak and possibly causing other issues. + + In an environment where writing to a pointer may not be atomic, the + above logic is not sufficient. The thread that is doing the compiling + may be descheduled after writing only part of the pointer, which could + cause other threads to use an invalid value. Instead of checking the + pointer itself, a separate "pointer is valid" flag (that can be updated + atomically) must be used: + + Get a read-only (shared) lock (mutex) for pointer + if (!pointer_is_valid) + { + Get a write (unique) lock for pointer + if (!pointer_is_valid) + { + pointer = pcre2_compile(... + pointer_is_valid = TRUE + } + } + Release the lock + Use pointer in pcre2_match() + If JIT is being used, but the JIT compilation is not being done immedi- - ately, (perhaps waiting to see if the pattern is used often enough) - similar logic is required. JIT compilation updates a pointer within the + ately (perhaps waiting to see if the pattern is used often enough), + similar logic is required. JIT compilation updates a value within the compiled code block, so a thread must gain unique write access to the pointer before calling pcre2_jit_compile(). Alternatively, - pcre2_code_copy() or pcre2_code_copy_with_tables() can be used to - obtain a private copy of the compiled code before calling the JIT com- + pcre2_code_copy() or pcre2_code_copy_with_tables() can be used to ob- + tain a private copy of the compiled code before calling the JIT com- piler. Context blocks @@ -736,9 +768,9 @@ PCRE2 CONTEXTS argument lists at a reasonable size, and at the same time to keep the API extensible, "uncommon" parameters are passed to certain functions in a context instead of directly. A context is just a block of memory - that holds the parameter values. Applications that do not need to - adjust any of the context parameters can pass NULL when a context - pointer is required. + that holds the parameter values. Applications that do not need to ad- + just any of the context parameters can pass NULL when a context pointer + is required. There are three different types of context: a general context that is relevant for several PCRE2 operations, a compile-time context, and a @@ -746,9 +778,9 @@ PCRE2 CONTEXTS The general context - At present, this context just contains pointers to (and data for) - external memory management functions that are called from several - places in the PCRE2 library. The context is named `general' rather than + At present, this context just contains pointers to (and data for) ex- + ternal memory management functions that are called from several places + in the PCRE2 library. The context is named `general' rather than specifically `memory' because in future other fields may be added. If you do not want to supply your own custom memory management functions, you do not need to bother with a general context. A general context is @@ -769,9 +801,9 @@ PCRE2 CONTEXTS function may be NULL, in which case the system memory management func- tions malloc() and free() are used. (This is not currently useful, as there are no other fields in a general context, but in future there - might be.) The private_malloc() function is used (if supplied) to - obtain memory for storing the context, and all three values are saved - as part of the context. + might be.) The private_malloc() function is used (if supplied) to ob- + tain memory for storing the context, and all three values are saved as + part of the context. Whenever PCRE2 creates a data block of any kind, the block contains a pointer to the free() function that matches the malloc() function that @@ -832,7 +864,7 @@ PCRE2 CONTEXTS pcre2_dfa_match(). int pcre2_set_character_tables(pcre2_compile_context *ccontext, - const unsigned char *tables); + const uint8_t *tables); The value must be the result of a call to pcre2_maketables(), whose only argument is a general context. This function builds a set of char- @@ -846,8 +878,8 @@ PCRE2 CONTEXTS avoid running out, the compile context contains a set of extra option bits which are used for some newer, assumed rarer, options. This func- tion sets those bits. It always sets all the bits (either on or off). - It does not modify any existing setting. The available options are - defined in the section entitled "Extra compile options" below. + It does not modify any existing setting. The available options are de- + fined in the section entitled "Extra compile options" below. int pcre2_set_max_pattern_length(pcre2_compile_context *ccontext, PCRE2_SIZE value); @@ -872,11 +904,11 @@ PCRE2 CONTEXTS A pattern can override the value set in the compile context by starting with a sequence such as (*CRLF). See the pcre2pattern page for details. - When a pattern is compiled with the PCRE2_EXTENDED or - PCRE2_EXTENDED_MORE option, the newline convention affects the recogni- - tion of the end of internal comments starting with #. The value is - saved with the compiled pattern for subsequent use by the JIT compiler - and by the two interpreted matching functions, pcre2_match() and + When a pattern is compiled with the PCRE2_EXTENDED or PCRE2_EX- + TENDED_MORE option, the newline convention affects the recognition of + the end of internal comments starting with #. The value is saved with + the compiled pattern for subsequent use by the JIT compiler and by the + two interpreted matching functions, pcre2_match() and pcre2_dfa_match(). int pcre2_set_parens_nest_limit(pcre2_compile_context *ccontext, @@ -953,24 +985,22 @@ PCRE2 CONTEXTS int pcre2_set_offset_limit(pcre2_match_context *mcontext, PCRE2_SIZE value); - The offset_limit parameter limits how far an unanchored search can - advance in the subject string. The default value is PCRE2_UNSET. The - pcre2_match() and pcre2_dfa_match() functions return - PCRE2_ERROR_NOMATCH if a match with a starting point before or at the - given offset is not found. The pcre2_substitute() function makes no - more substitutions. - - For example, if the pattern /abc/ is matched against "123abc" with an - offset limit less than 3, the result is PCRE2_ERROR_NOMATCH. A match - can never be found if the startoffset argument of pcre2_match(), - pcre2_dfa_match(), or pcre2_substitute() is greater than the offset + The offset_limit parameter limits how far an unanchored search can ad- + vance in the subject string. The default value is PCRE2_UNSET. The + pcre2_match() and pcre2_dfa_match() functions return PCRE2_ERROR_NO- + MATCH if a match with a starting point before or at the given offset is + not found. The pcre2_substitute() function makes no more substitutions. + + For example, if the pattern /abc/ is matched against "123abc" with an + offset limit less than 3, the result is PCRE2_ERROR_NOMATCH. A match + can never be found if the startoffset argument of pcre2_match(), + pcre2_dfa_match(), or pcre2_substitute() is greater than the offset limit set in the match context. - When using this facility, you must set the PCRE2_USE_OFFSET_LIMIT - option when calling pcre2_compile() so that when JIT is in use, differ- - ent code can be compiled. If a match is started with a non-default - match limit when PCRE2_USE_OFFSET_LIMIT is not set, an error is gener- - ated. + When using this facility, you must set the PCRE2_USE_OFFSET_LIMIT op- + tion when calling pcre2_compile() so that when JIT is in use, different + code can be compiled. If a match is started with a non-default match + limit when PCRE2_USE_OFFSET_LIMIT is not set, an error is generated. The offset limit facility can be used to track progress when searching large subject strings or to limit the extent of global substitutions. @@ -1000,9 +1030,9 @@ PCRE2 CONTEXTS (*LIMIT_HEAP=ddd) - where ddd is a decimal number. However, such a setting is ignored - unless ddd is less than the limit set by the caller of pcre2_match() - or, if no such limit is set, less than the default. + where ddd is a decimal number. However, such a setting is ignored un- + less ddd is less than the limit set by the caller of pcre2_match() or, + if no such limit is set, less than the default. The pcre2_match() function starts out using a 20KiB vector on the sys- tem stack for recording backtracking points. The more nested backtrack- @@ -1021,8 +1051,8 @@ PCRE2 CONTEXTS int pcre2_set_match_limit(pcre2_match_context *mcontext, uint32_t value); - The match_limit parameter provides a means of preventing PCRE2 from - using up too many computing resources when processing patterns that are + The match_limit parameter provides a means of preventing PCRE2 from us- + ing up too many computing resources when processing patterns that are not going to match, but which have a very large number of possibilities in their search trees. The classic example is a pattern that uses nested unlimited repeats. @@ -1042,15 +1072,15 @@ PCRE2 CONTEXTS value is also used in this case (but in a different way) to limit how long the matching can continue. - The default value for the limit can be set when PCRE2 is built; the - default default is 10 million, which handles all but the most extreme + The default value for the limit can be set when PCRE2 is built; the de- + fault default is 10 million, which handles all but the most extreme cases. A value for the match limit may also be supplied by an item at the start of a pattern of the form (*LIMIT_MATCH=ddd) - where ddd is a decimal number. However, such a setting is ignored - unless ddd is less than the limit set by the caller of pcre2_match() or + where ddd is a decimal number. However, such a setting is ignored un- + less ddd is less than the limit set by the caller of pcre2_match() or pcre2_dfa_match() or, if no such limit is set, less than the default. int pcre2_set_depth_limit(pcre2_match_context *mcontext, @@ -1060,8 +1090,8 @@ PCRE2 CONTEXTS pcre2_match(). Each time a nested backtracking point is passed, a new memory "frame" is used to remember the state of matching at that point. Thus, this parameter indirectly limits the amount of memory that is - used in a match. However, because the size of each memory "frame" - depends on the number of capturing parentheses, the actual memory limit + used in a match. However, because the size of each memory "frame" de- + pends on the number of capturing parentheses, the actual memory limit varies from pattern to pattern. This limit was more useful in versions before 10.30, where function recursion was used for backtracking. @@ -1076,13 +1106,13 @@ PCRE2 CONTEXTS each call uses only a few hundred bytes, even a small stack can support quite a lot of recursion. - If the depth of internal recursive function calls is great enough, - local workspace vectors are allocated on the heap from version 10.32 - onwards, so the depth limit also indirectly limits the amount of heap + If the depth of internal recursive function calls is great enough, lo- + cal workspace vectors are allocated on the heap from version 10.32 on- + wards, so the depth limit also indirectly limits the amount of heap memory that is used. A recursive pattern such as /(.(?2))((?1)|)/, when matched to a very long string using pcre2_dfa_match(), can use a great - deal of memory. However, it is probably better to limit heap usage - directly by calling pcre2_set_heap_limit(). + deal of memory. However, it is probably better to limit heap usage di- + rectly by calling pcre2_set_heap_limit(). The default value for the depth limit can be set when PCRE2 is built; if it is not, the default is set to the same value as the default for @@ -1093,8 +1123,8 @@ PCRE2 CONTEXTS (*LIMIT_DEPTH=ddd) - where ddd is a decimal number. However, such a setting is ignored - unless ddd is less than the limit set by the caller of pcre2_match() or + where ddd is a decimal number. However, such a setting is ignored un- + less ddd is less than the limit set by the caller of pcre2_match() or pcre2_dfa_match() or, if no such limit is set, less than the default. @@ -1103,18 +1133,18 @@ CHECKING BUILD-TIME OPTIONS int pcre2_config(uint32_t what, void *where); The function pcre2_config() makes it possible for a PCRE2 client to - discover which optional features have been compiled into the PCRE2 - library. The pcre2build documentation has more details about these - optional features. + find the value of certain configuration parameters and to discover + which optional features have been compiled into the PCRE2 library. The + pcre2build documentation has more details about these features. The first argument for pcre2_config() specifies which information is - required. The second argument is a pointer to memory into which the - information is placed. If NULL is passed, the function returns the - amount of memory that is needed for the requested information. For - calls that return numerical values, the value is in bytes; when - requesting these values, where should point to appropriately aligned - memory. For calls that return strings, the required length is given in - code units, not counting the terminating zero. + required. The second argument is a pointer to memory into which the in- + formation is placed. If NULL is passed, the function returns the amount + of memory that is needed for the requested information. For calls that + return numerical values, the value is in bytes; when requesting these + values, where should point to appropriately aligned memory. For calls + that return strings, the required length is given in code units, not + counting the terminating zero. When requesting information, the returned value from pcre2_config() is non-negative on success, or the negative error code PCRE2_ERROR_BADOP- @@ -1125,9 +1155,9 @@ CHECKING BUILD-TIME OPTIONS The output is a uint32_t integer whose value indicates what character sequences the \R escape sequence matches by default. A value of - PCRE2_BSR_UNICODE means that \R matches any Unicode line ending - sequence; a value of PCRE2_BSR_ANYCRLF means that \R matches only CR, - LF, or CRLF. The default can be overridden when a pattern is compiled. + PCRE2_BSR_UNICODE means that \R matches any Unicode line ending se- + quence; a value of PCRE2_BSR_ANYCRLF means that \R matches only CR, LF, + or CRLF. The default can be overridden when a pattern is compiled. PCRE2_CONFIG_COMPILED_WIDTHS @@ -1161,8 +1191,8 @@ CHECKING BUILD-TIME OPTIONS units long. (The exact length required can be found by calling pcre2_config() with where set to NULL.) The buffer is filled with a string that contains the name of the architecture for which the JIT - compiler is configured, for example "x86 32bit (little endian + - unaligned)". If JIT support is not available, PCRE2_ERROR_BADOPTION is + compiler is configured, for example "x86 32bit (little endian + un- + aligned)". If JIT support is not available, PCRE2_ERROR_BADOPTION is returned, otherwise the number of code units used is returned. This is the length of the string, plus one unit for the terminating zero. @@ -1216,8 +1246,8 @@ CHECKING BUILD-TIME OPTIONS ing of parentheses (of any kind) in a pattern. This limit is imposed to cap the amount of system stack used when a pattern is compiled. It is specified when PCRE2 is built; the default is 250. This limit does not - take into account the stack that may already be used by the calling - application. For finer control over compilation stack usage, see + take into account the stack that may already be used by the calling ap- + plication. For finer control over compilation stack usage, see pcre2_set_compile_recursion_guard(). PCRE2_CONFIG_STACKRECURSE @@ -1225,6 +1255,12 @@ CHECKING BUILD-TIME OPTIONS This parameter is obsolete and should not be used in new code. The out- put is a uint32_t integer that is always set to zero. + PCRE2_CONFIG_TABLES_LENGTH + + The output is a uint32_t integer that gives the length of PCRE2's char- + acter processing tables in bytes. For details of these tables see the + section on locale support below. + PCRE2_CONFIG_UNICODE_VERSION The where argument should point to a buffer that is at least 24 code @@ -1267,21 +1303,21 @@ COMPILING A PATTERN The pattern is defined by a pointer to a string of code units and a length (in code units). If the pattern is zero-terminated, the length can be specified as PCRE2_ZERO_TERMINATED. The function returns a - pointer to a block of memory that contains the compiled pattern and - related data, or NULL if an error occurred. + pointer to a block of memory that contains the compiled pattern and re- + lated data, or NULL if an error occurred. If the compile context argument ccontext is NULL, memory for the com- - piled pattern is obtained by calling malloc(). Otherwise, it is - obtained from the same memory function that was used for the compile - context. The caller must free the memory by calling pcre2_code_free() - when it is no longer needed. If pcre2_code_free() is called with a - NULL argument, it returns immediately, without doing anything. + piled pattern is obtained by calling malloc(). Otherwise, it is ob- + tained from the same memory function that was used for the compile con- + text. The caller must free the memory by calling pcre2_code_free() when + it is no longer needed. If pcre2_code_free() is called with a NULL ar- + gument, it returns immediately, without doing anything. The function pcre2_code_copy() makes a copy of the compiled code in new memory, using the same memory allocator as was used for the original. - However, if the code has been processed by the JIT compiler (see - below), the JIT information cannot be copied (because it is position- - dependent). The new copy can initially be used only for non-JIT match- + However, if the code has been processed by the JIT compiler (see be- + low), the JIT information cannot be copied (because it is position-de- + pendent). The new copy can initially be used only for non-JIT match- ing, though it can be passed to pcre2_jit_compile() if required. If pcre2_code_copy() is called with a NULL argument, it returns NULL. @@ -1307,15 +1343,15 @@ COMPILING A PATTERN compiled pattern or a subject string until after all operations on the match data block have taken place, unless, in the case of the subject string, you have used the PCRE2_COPY_MATCHED_SUBJECT option, which is - described in the section entitled "Option bits for pcre2_match()" - below. + described in the section entitled "Option bits for pcre2_match()" be- + low. The options argument for pcre2_compile() contains various bit settings - that affect the compilation. It should be zero if none of them are - required. The available options are described below. Some of them (in + that affect the compilation. It should be zero if none of them are re- + quired. The available options are described below. Some of them (in particular, those that are compatible with Perl, but some others as - well) can also be set and unset from within the pattern (see the - detailed description in the pcre2pattern documentation). + well) can also be set and unset from within the pattern (see the de- + tailed description in the pcre2pattern documentation). For those options that can be different in different parts of the pat- tern, the contents of the options argument specifies their settings at @@ -1323,42 +1359,43 @@ COMPILING A PATTERN PCRE2_NO_UTF_CHECK options can be set at the time of matching as well as at compile time. - Some additional options and less frequently required compile-time - parameters (for example, the newline setting) can be provided in a com- + Some additional options and less frequently required compile-time pa- + rameters (for example, the newline setting) can be provided in a com- pile context (as described above). If errorcode or erroroffset is NULL, pcre2_compile() returns NULL imme- diately. Otherwise, the variables to which these point are set to an - error code and an offset (number of code units) within the pattern, - respectively, when pcre2_compile() returns NULL because a compilation - error has occurred. The values are not defined when compilation is suc- + error code and an offset (number of code units) within the pattern, re- + spectively, when pcre2_compile() returns NULL because a compilation er- + ror has occurred. The values are not defined when compilation is suc- cessful and pcre2_compile() returns a non-NULL value. - There are nearly 100 positive error codes that pcre2_compile() may - return if it finds an error in the pattern. There are also some nega- - tive error codes that are used for invalid UTF strings. These are the - same as given by pcre2_match() and pcre2_dfa_match(), and are described - in the pcre2unicode page. There is no separate documentation for the - positive error codes, because the textual error messages that are - obtained by calling the pcre2_get_error_message() function (see - "Obtaining a textual error message" below) should be self-explanatory. - Macro names starting with PCRE2_ERROR_ are defined for both positive - and negative error codes in pcre2.h. + There are nearly 100 positive error codes that pcre2_compile() may re- + turn if it finds an error in the pattern. There are also some negative + error codes that are used for invalid UTF strings when validity check- + ing is in force. These are the same as given by pcre2_match() and + pcre2_dfa_match(), and are described in the pcre2unicode documentation. + There is no separate documentation for the positive error codes, be- + cause the textual error messages that are obtained by calling the + pcre2_get_error_message() function (see "Obtaining a textual error mes- + sage" below) should be self-explanatory. Macro names starting with + PCRE2_ERROR_ are defined for both positive and negative error codes in + pcre2.h. The value returned in erroroffset is an indication of where in the pat- - tern the error occurred. It is not necessarily the furthest point in - the pattern that was read. For example, after the error "lookbehind - assertion is not fixed length", the error offset points to the start of - the failing assertion. For an invalid UTF-8 or UTF-16 string, the off- + tern the error occurred. It is not necessarily the furthest point in + the pattern that was read. For example, after the error "lookbehind as- + sertion is not fixed length", the error offset points to the start of + the failing assertion. For an invalid UTF-8 or UTF-16 string, the off- set is that of the first code unit of the failing character. - Some errors are not detected until the whole pattern has been scanned; - in these cases, the offset passed back is the length of the pattern. - Note that the offset is in code units, not characters, even in a UTF + Some errors are not detected until the whole pattern has been scanned; + in these cases, the offset passed back is the length of the pattern. + Note that the offset is in code units, not characters, even in a UTF mode. It may sometimes point into the middle of a UTF-8 or UTF-16 char- acter. - This code fragment shows a typical straightforward call to pcre2_com- + This code fragment shows a typical straightforward call to pcre2_com- pile(): pcre2_code *re; @@ -1375,92 +1412,95 @@ COMPILING A PATTERN Main compile options - The following names for option bits are defined in the pcre2.h header + The following names for option bits are defined in the pcre2.h header file: PCRE2_ANCHORED If this bit is set, the pattern is forced to be "anchored", that is, it - is constrained to match only at the first matching point in the string - that is being searched (the "subject string"). This effect can also be - achieved by appropriate constructs in the pattern itself, which is the + is constrained to match only at the first matching point in the string + that is being searched (the "subject string"). This effect can also be + achieved by appropriate constructs in the pattern itself, which is the only way to do it in Perl. PCRE2_ALLOW_EMPTY_CLASS - By default, for compatibility with Perl, a closing square bracket that - immediately follows an opening one is treated as a data character for - the class. When PCRE2_ALLOW_EMPTY_CLASS is set, it terminates the + By default, for compatibility with Perl, a closing square bracket that + immediately follows an opening one is treated as a data character for + the class. When PCRE2_ALLOW_EMPTY_CLASS is set, it terminates the class, which therefore contains no characters and so can never match. PCRE2_ALT_BSUX - This option request alternative handling of three escape sequences, - which makes PCRE2's behaviour more like ECMAscript (aka JavaScript). + This option request alternative handling of three escape sequences, + which makes PCRE2's behaviour more like ECMAscript (aka JavaScript). When it is set: (1) \U matches an upper case "U" character; by default \U causes a com- pile time error (Perl uses \U to upper case subsequent characters). (2) \u matches a lower case "u" character unless it is followed by four - hexadecimal digits, in which case the hexadecimal number defines the - code point to match. By default, \u causes a compile time error (Perl + hexadecimal digits, in which case the hexadecimal number defines the + code point to match. By default, \u causes a compile time error (Perl uses it to upper case the following character). - (3) \x matches a lower case "x" character unless it is followed by two - hexadecimal digits, in which case the hexadecimal number defines the - code point to match. By default, as in Perl, a hexadecimal number is + (3) \x matches a lower case "x" character unless it is followed by two + hexadecimal digits, in which case the hexadecimal number defines the + code point to match. By default, as in Perl, a hexadecimal number is always expected after \x, but it may have zero, one, or two digits (so, for example, \xz matches a binary zero character followed by z). ECMAscript 6 added additional functionality to \u. This can be accessed - using the PCRE2_EXTRA_ALT_BSUX extra option (see "Extra compile - options" below). Note that this alternative escape handling applies - only to patterns. Neither of these options affects the processing of - replacement strings passed to pcre2_substitute(). + using the PCRE2_EXTRA_ALT_BSUX extra option (see "Extra compile op- + tions" below). Note that this alternative escape handling applies only + to patterns. Neither of these options affects the processing of re- + placement strings passed to pcre2_substitute(). PCRE2_ALT_CIRCUMFLEX In multiline mode (when PCRE2_MULTILINE is set), the circumflex - metacharacter matches at the start of the subject (unless PCRE2_NOTBOL - is set), and also after any internal newline. However, it does not + metacharacter matches at the start of the subject (unless PCRE2_NOTBOL + is set), and also after any internal newline. However, it does not match after a newline at the end of the subject, for compatibility with - Perl. If you want a multiline circumflex also to match after a termi- + Perl. If you want a multiline circumflex also to match after a termi- nating newline, you must set PCRE2_ALT_CIRCUMFLEX. PCRE2_ALT_VERBNAMES - By default, for compatibility with Perl, the name in any verb sequence - such as (*MARK:NAME) is any sequence of characters that does not - include a closing parenthesis. The name is not processed in any way, - and it is not possible to include a closing parenthesis in the name. - However, if the PCRE2_ALT_VERBNAMES option is set, normal backslash - processing is applied to verb names and only an unescaped closing - parenthesis terminates the name. A closing parenthesis can be included - in a name either as \) or between \Q and \E. If the PCRE2_EXTENDED or - PCRE2_EXTENDED_MORE option is set with PCRE2_ALT_VERBNAMES, unescaped - whitespace in verb names is skipped and #-comments are recognized, - exactly as in the rest of the pattern. + By default, for compatibility with Perl, the name in any verb sequence + such as (*MARK:NAME) is any sequence of characters that does not in- + clude a closing parenthesis. The name is not processed in any way, and + it is not possible to include a closing parenthesis in the name. How- + ever, if the PCRE2_ALT_VERBNAMES option is set, normal backslash pro- + cessing is applied to verb names and only an unescaped closing paren- + thesis terminates the name. A closing parenthesis can be included in a + name either as \) or between \Q and \E. If the PCRE2_EXTENDED or + PCRE2_EXTENDED_MORE option is set with PCRE2_ALT_VERBNAMES, unescaped + whitespace in verb names is skipped and #-comments are recognized, ex- + actly as in the rest of the pattern. PCRE2_AUTO_CALLOUT - If this bit is set, pcre2_compile() automatically inserts callout - items, all with number 255, before each pattern item, except immedi- - ately before or after an explicit callout in the pattern. For discus- + If this bit is set, pcre2_compile() automatically inserts callout + items, all with number 255, before each pattern item, except immedi- + ately before or after an explicit callout in the pattern. For discus- sion of the callout facility, see the pcre2callout documentation. PCRE2_CASELESS - If this bit is set, letters in the pattern match both upper and lower - case letters in the subject. It is equivalent to Perl's /i option, and - it can be changed within a pattern by a (?i) option setting. If - PCRE2_UTF is set, Unicode properties are used for all characters with - more than one other case, and for all characters whose code points are - greater than U+007F. For lower valued characters with only one other - case, a lookup table is used for speed. When PCRE2_UTF is not set, a - lookup table is used for all code points less than 256, and higher code - points (available only in 16-bit or 32-bit mode) are treated as not - having another case. + If this bit is set, letters in the pattern match both upper and lower + case letters in the subject. It is equivalent to Perl's /i option, and + it can be changed within a pattern by a (?i) option setting. If either + PCRE2_UTF or PCRE2_UCP is set, Unicode properties are used for all + characters with more than one other case, and for all characters whose + code points are greater than U+007F. Note that there are two ASCII + characters, K and S, that, in addition to their lower case ASCII equiv- + alents, are case-equivalent with U+212A (Kelvin sign) and U+017F (long + S) respectively. For lower valued characters with only one other case, + a lookup table is used for speed. When neither PCRE2_UTF nor PCRE2_UCP + is set, a lookup table is used for all code points less than 256, and + higher code points (available only in 16-bit or 32-bit mode) are + treated as not having another case. PCRE2_DOLLAR_ENDONLY @@ -1517,56 +1557,54 @@ COMPILING A PATTERN PCRE2_EXTENDED - If this bit is set, most white space characters in the pattern are - totally ignored except when escaped or inside a character class. How- - ever, white space is not allowed within sequences such as (?> that - introduce various parenthesized groups, nor within numerical quanti- - fiers such as {1,3}. Ignorable white space is permitted between an item - and a following quantifier and between a quantifier and a following + - that indicates possessiveness. PCRE2_EXTENDED is equivalent to Perl's - /x option, and it can be changed within a pattern by a (?x) option set- - ting. - - When PCRE2 is compiled without Unicode support, PCRE2_EXTENDED recog- - nizes as white space only those characters with code points less than + If this bit is set, most white space characters in the pattern are to- + tally ignored except when escaped or inside a character class. However, + white space is not allowed within sequences such as (?> that introduce + various parenthesized groups, nor within numerical quantifiers such as + {1,3}. Ignorable white space is permitted between an item and a follow- + ing quantifier and between a quantifier and a following + that indi- + cates possessiveness. PCRE2_EXTENDED is equivalent to Perl's /x option, + and it can be changed within a pattern by a (?x) option setting. + + When PCRE2 is compiled without Unicode support, PCRE2_EXTENDED recog- + nizes as white space only those characters with code points less than 256 that are flagged as white space in its low-character table. The ta- ble is normally created by pcre2_maketables(), which uses the isspace() - function to identify space characters. In most ASCII environments, the - relevant characters are those with code points 0x0009 (tab), 0x000A - (linefeed), 0x000B (vertical tab), 0x000C (formfeed), 0x000D (carriage + function to identify space characters. In most ASCII environments, the + relevant characters are those with code points 0x0009 (tab), 0x000A + (linefeed), 0x000B (vertical tab), 0x000C (formfeed), 0x000D (carriage return), and 0x0020 (space). When PCRE2 is compiled with Unicode support, in addition to these char- - acters, five more Unicode "Pattern White Space" characters are recog- + acters, five more Unicode "Pattern White Space" characters are recog- nized by PCRE2_EXTENDED. These are U+0085 (next line), U+200E (left-to- - right mark), U+200F (right-to-left mark), U+2028 (line separator), and - U+2029 (paragraph separator). This set of characters is the same as - recognized by Perl's /x option. Note that the horizontal and vertical - space characters that are matched by the \h and \v escapes in patterns + right mark), U+200F (right-to-left mark), U+2028 (line separator), and + U+2029 (paragraph separator). This set of characters is the same as + recognized by Perl's /x option. Note that the horizontal and vertical + space characters that are matched by the \h and \v escapes in patterns are a much bigger set. - As well as ignoring most white space, PCRE2_EXTENDED also causes char- - acters between an unescaped # outside a character class and the next - newline, inclusive, to be ignored, which makes it possible to include + As well as ignoring most white space, PCRE2_EXTENDED also causes char- + acters between an unescaped # outside a character class and the next + newline, inclusive, to be ignored, which makes it possible to include comments inside complicated patterns. Note that the end of this type of - comment is a literal newline sequence in the pattern; escape sequences + comment is a literal newline sequence in the pattern; escape sequences that happen to represent a newline do not count. Which characters are interpreted as newlines can be specified by a set- - ting in the compile context that is passed to pcre2_compile() or by a - special sequence at the start of the pattern, as described in the sec- - tion entitled "Newline conventions" in the pcre2pattern documentation. + ting in the compile context that is passed to pcre2_compile() or by a + special sequence at the start of the pattern, as described in the sec- + tion entitled "Newline conventions" in the pcre2pattern documentation. A default is defined when PCRE2 is built. PCRE2_EXTENDED_MORE - This option has the effect of PCRE2_EXTENDED, but, in addition, - unescaped space and horizontal tab characters are ignored inside a - character class. Note: only these two characters are ignored, not the - full set of pattern white space characters that are ignored outside a - character class. PCRE2_EXTENDED_MORE is equivalent to Perl's /xx - option, and it can be changed within a pattern by a (?xx) option set- - ting. + This option has the effect of PCRE2_EXTENDED, but, in addition, un- + escaped space and horizontal tab characters are ignored inside a char- + acter class. Note: only these two characters are ignored, not the full + set of pattern white space characters that are ignored outside a char- + acter class. PCRE2_EXTENDED_MORE is equivalent to Perl's /xx option, + and it can be changed within a pattern by a (?xx) option setting. PCRE2_FIRSTLINE @@ -1591,52 +1629,58 @@ COMPILING A PATTERN ciency, you should consider using other approaches. The only other main options that are allowed with PCRE2_LITERAL are: PCRE2_ANCHORED, PCRE2_ENDANCHORED, PCRE2_AUTO_CALLOUT, PCRE2_CASELESS, PCRE2_FIRSTLINE, - PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, PCRE2_UTF, and - PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EXTRA_MATCH_LINE and - PCRE2_EXTRA_MATCH_WORD are also supported. Any other options cause an - error. + PCRE2_MATCH_INVALID_UTF, PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, + PCRE2_UTF, and PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EX- + TRA_MATCH_LINE and PCRE2_EXTRA_MATCH_WORD are also supported. Any other + options cause an error. + + PCRE2_MATCH_INVALID_UTF + + This option forces PCRE2_UTF (see below) and also enables support for + matching by pcre2_match() in subject strings that contain invalid UTF + sequences. This facility is not supported for DFA matching. For de- + tails, see the pcre2unicode documentation. PCRE2_MATCH_UNSET_BACKREF - If this option is set, a backreference to an unset capture group - matches an empty string (by default this causes the current matching - alternative to fail). A pattern such as (\1)(a) succeeds when this - option is set (assuming it can find an "a" in the subject), whereas it - fails by default, for Perl compatibility. Setting this option makes + If this option is set, a backreference to an unset capture group + matches an empty string (by default this causes the current matching + alternative to fail). A pattern such as (\1)(a) succeeds when this op- + tion is set (assuming it can find an "a" in the subject), whereas it + fails by default, for Perl compatibility. Setting this option makes PCRE2 behave more like ECMAscript (aka JavaScript). PCRE2_MULTILINE - By default, for the purposes of matching "start of line" and "end of - line", PCRE2 treats the subject string as consisting of a single line - of characters, even if it actually contains newlines. The "start of - line" metacharacter (^) matches only at the start of the string, and - the "end of line" metacharacter ($) matches only at the end of the - string, or before a terminating newline (except when PCRE2_DOL- - LAR_ENDONLY is set). Note, however, that unless PCRE2_DOTALL is set, - the "any character" metacharacter (.) does not match at a newline. This - behaviour (for ^, $, and dot) is the same as Perl. - - When PCRE2_MULTILINE it is set, the "start of line" and "end of line" - constructs match immediately following or immediately before internal - newlines in the subject string, respectively, as well as at the very - start and end. This is equivalent to Perl's /m option, and it can be + By default, for the purposes of matching "start of line" and "end of + line", PCRE2 treats the subject string as consisting of a single line + of characters, even if it actually contains newlines. The "start of + line" metacharacter (^) matches only at the start of the string, and + the "end of line" metacharacter ($) matches only at the end of the + string, or before a terminating newline (except when PCRE2_DOLLAR_EN- + DONLY is set). Note, however, that unless PCRE2_DOTALL is set, the "any + character" metacharacter (.) does not match at a newline. This behav- + iour (for ^, $, and dot) is the same as Perl. + + When PCRE2_MULTILINE it is set, the "start of line" and "end of line" + constructs match immediately following or immediately before internal + newlines in the subject string, respectively, as well as at the very + start and end. This is equivalent to Perl's /m option, and it can be changed within a pattern by a (?m) option setting. Note that the "start of line" metacharacter does not match after a newline at the end of the - subject, for compatibility with Perl. However, you can change this by - setting the PCRE2_ALT_CIRCUMFLEX option. If there are no newlines in a - subject string, or no occurrences of ^ or $ in a pattern, setting + subject, for compatibility with Perl. However, you can change this by + setting the PCRE2_ALT_CIRCUMFLEX option. If there are no newlines in a + subject string, or no occurrences of ^ or $ in a pattern, setting PCRE2_MULTILINE has no effect. PCRE2_NEVER_BACKSLASH_C - This option locks out the use of \C in the pattern that is being com- - piled. This escape can cause unpredictable behaviour in UTF-8 or - UTF-16 modes, because it may leave the current matching point in the - middle of a multi-code-unit character. This option may be useful in - applications that process patterns from external sources. Note that - there is also a build-time option that permanently locks out the use of - \C. + This option locks out the use of \C in the pattern that is being com- + piled. This escape can cause unpredictable behaviour in UTF-8 or + UTF-16 modes, because it may leave the current matching point in the + middle of a multi-code-unit character. This option may be useful in ap- + plications that process patterns from external sources. Note that there + is also a build-time option that permanently locks out the use of \C. PCRE2_NEVER_UCP @@ -1653,9 +1697,9 @@ COMPILING A PATTERN This option locks out interpretation of the pattern as UTF-8, UTF-16, or UTF-32, depending on which library is in use. In particular, it pre- vents the creator of the pattern from switching to UTF interpretation - by starting the pattern with (*UTF). This option may be useful in - applications that process patterns from external sources. The combina- - tion of PCRE2_UTF and PCRE2_NEVER_UTF causes an error. + by starting the pattern with (*UTF). This option may be useful in ap- + plications that process patterns from external sources. The combination + of PCRE2_UTF and PCRE2_NEVER_UTF causes an error. PCRE2_NO_AUTO_CAPTURE @@ -1730,111 +1774,121 @@ COMPILING A PATTERN does. However, if the same match is run with PCRE2_NO_START_OPTIMIZE set, the initial scan along the subject string does not happen. The first match attempt is run starting from "D" and when this fails, - (*COMMIT) prevents any further matches being tried, so the overall - result is "no match". - - There are also other start-up optimizations. For example, a minimum - length for the subject may be recorded. Consider the pattern - - (*MARK:A)(X|Y) - - The minimum length for a match is one character. If the subject is - "ABC", there will be attempts to match "ABC", "BC", and "C". An attempt - to match an empty string at the end of the subject does not take place, - because PCRE2 knows that the subject is now too short, and so the - (*MARK) is never encountered. In this case, the optimization does not - affect the overall match result, which is still "no match", but it does - affect the auxiliary information that is returned. + (*COMMIT) prevents any further matches being tried, so the overall re- + sult is "no match". + + As another start-up optimization makes use of a minimum length for a + matching subject, which is recorded when possible. Consider the pattern + + (*MARK:1)B(*MARK:2)(X|Y) + + The minimum length for a match is two characters. If the subject is + "XXBB", the "starting character" optimization skips "XX", then tries to + match "BB", which is long enough. In the process, (*MARK:2) is encoun- + tered and remembered. When the match attempt fails, the next "B" is + found, but there is only one character left, so there are no more at- + tempts, and "no match" is returned with the "last mark seen" set to + "2". If NO_START_OPTIMIZE is set, however, matches are tried at every + possible starting position, including at the end of the subject, where + (*MARK:1) is encountered, but there is no "B", so the "last mark seen" + that is returned is "1". In this case, the optimizations do not affect + the overall match result, which is still "no match", but they do affect + the auxiliary information that is returned. PCRE2_NO_UTF_CHECK - When PCRE2_UTF is set, the validity of the pattern as a UTF string is - automatically checked. There are discussions about the validity of - UTF-8 strings, UTF-16 strings, and UTF-32 strings in the pcre2unicode - document. If an invalid UTF sequence is found, pcre2_compile() returns + When PCRE2_UTF is set, the validity of the pattern as a UTF string is + automatically checked. There are discussions about the validity of + UTF-8 strings, UTF-16 strings, and UTF-32 strings in the pcre2unicode + document. If an invalid UTF sequence is found, pcre2_compile() returns a negative error code. - If you know that your pattern is a valid UTF string, and you want to - skip this check for performance reasons, you can set the - PCRE2_NO_UTF_CHECK option. When it is set, the effect of passing an - invalid UTF string as a pattern is undefined. It may cause your program + If you know that your pattern is a valid UTF string, and you want to + skip this check for performance reasons, you can set the + PCRE2_NO_UTF_CHECK option. When it is set, the effect of passing an in- + valid UTF string as a pattern is undefined. It may cause your program to crash or loop. Note that this option can also be passed to pcre2_match() and - pcre_dfa_match(), to suppress UTF validity checking of the subject + pcre_dfa_match(), to suppress UTF validity checking of the subject string. Note also that setting PCRE2_NO_UTF_CHECK at compile time does not dis- - able the error that is given if an escape sequence for an invalid Uni- - code code point is encountered in the pattern. In particular, the so- - called "surrogate" code points (0xd800 to 0xdfff) are invalid. If you - want to allow escape sequences such as \x{d800} you can set the - PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES extra option, as described in the - section entitled "Extra compile options" below. However, this is pos- + able the error that is given if an escape sequence for an invalid Uni- + code code point is encountered in the pattern. In particular, the so- + called "surrogate" code points (0xd800 to 0xdfff) are invalid. If you + want to allow escape sequences such as \x{d800} you can set the + PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES extra option, as described in the + section entitled "Extra compile options" below. However, this is pos- sible only in UTF-8 and UTF-32 modes, because these values are not rep- resentable in UTF-16. PCRE2_UCP - This option changes the way PCRE2 processes \B, \b, \D, \d, \S, \s, \W, - \w, and some of the POSIX character classes. By default, only ASCII - characters are recognized, but if PCRE2_UCP is set, Unicode properties - are used instead to classify characters. More details are given in the - section on generic character types in the pcre2pattern page. If you set - PCRE2_UCP, matching one of the items it affects takes much longer. The - option is available only if PCRE2 has been compiled with Unicode sup- - port (which is the default). + This option has two effects. Firstly, it change the way PCRE2 processes + \B, \b, \D, \d, \S, \s, \W, \w, and some of the POSIX character + classes. By default, only ASCII characters are recognized, but if + PCRE2_UCP is set, Unicode properties are used instead to classify char- + acters. More details are given in the section on generic character + types in the pcre2pattern page. If you set PCRE2_UCP, matching one of + the items it affects takes much longer. + + The second effect of PCRE2_UCP is to force the use of Unicode proper- + ties for upper/lower casing operations on characters with code points + greater than 127, even when PCRE2_UTF is not set. This makes it possi- + ble, for example, to process strings in the 16-bit UCS-2 code. This op- + tion is available only if PCRE2 has been compiled with Unicode support + (which is the default). PCRE2_UNGREEDY - This option inverts the "greediness" of the quantifiers so that they - are not greedy by default, but become greedy if followed by "?". It is - not compatible with Perl. It can also be set by a (?U) option setting + This option inverts the "greediness" of the quantifiers so that they + are not greedy by default, but become greedy if followed by "?". It is + not compatible with Perl. It can also be set by a (?U) option setting within the pattern. PCRE2_USE_OFFSET_LIMIT This option must be set for pcre2_compile() if pcre2_set_offset_limit() - is going to be used to set a non-default offset limit in a match con- - text for matches that use this pattern. An error is generated if an - offset limit is set without this option. For more details, see the - description of pcre2_set_offset_limit() in the section that describes + is going to be used to set a non-default offset limit in a match con- + text for matches that use this pattern. An error is generated if an + offset limit is set without this option. For more details, see the de- + scription of pcre2_set_offset_limit() in the section that describes match contexts. See also the PCRE2_FIRSTLINE option above. PCRE2_UTF - This option causes PCRE2 to regard both the pattern and the subject - strings that are subsequently processed as strings of UTF characters - instead of single-code-unit strings. It is available when PCRE2 is - built to include Unicode support (which is the default). If Unicode - support is not available, the use of this option provokes an error. - Details of how PCRE2_UTF changes the behaviour of PCRE2 are given in - the pcre2unicode page. In particular, note that it changes the way + This option causes PCRE2 to regard both the pattern and the subject + strings that are subsequently processed as strings of UTF characters + instead of single-code-unit strings. It is available when PCRE2 is + built to include Unicode support (which is the default). If Unicode + support is not available, the use of this option provokes an error. De- + tails of how PCRE2_UTF changes the behaviour of PCRE2 are given in the + pcre2unicode page. In particular, note that it changes the way PCRE2_CASELESS handles characters with code points greater than 127. Extra compile options - The option bits that can be set in a compile context by calling the + The option bits that can be set in a compile context by calling the pcre2_set_compile_extra_options() function are as follows: PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES - This option applies when compiling a pattern in UTF-8 or UTF-32 mode. - It is forbidden in UTF-16 mode, and ignored in non-UTF modes. Unicode + This option applies when compiling a pattern in UTF-8 or UTF-32 mode. + It is forbidden in UTF-16 mode, and ignored in non-UTF modes. Unicode "surrogate" code points in the range 0xd800 to 0xdfff are used in pairs - in UTF-16 to encode code points with values in the range 0x10000 to - 0x10ffff. The surrogates cannot therefore be represented in UTF-16. + in UTF-16 to encode code points with values in the range 0x10000 to + 0x10ffff. The surrogates cannot therefore be represented in UTF-16. They can be represented in UTF-8 and UTF-32, but are defined as invalid - code points, and cause errors if encountered in a UTF-8 or UTF-32 + code points, and cause errors if encountered in a UTF-8 or UTF-32 string that is being checked for validity by PCRE2. - These values also cause errors if encountered in escape sequences such + These values also cause errors if encountered in escape sequences such as \x{d912} within a pattern. However, it seems that some applications, - when using PCRE2 to check for unwanted characters in UTF-8 strings, - explicitly test for the surrogates using escape sequences. The - PCRE2_NO_UTF_CHECK option does not disable the error that occurs, - because it applies only to the testing of input strings for UTF valid- - ity. + when using PCRE2 to check for unwanted characters in UTF-8 strings, ex- + plicitly test for the surrogates using escape sequences. The + PCRE2_NO_UTF_CHECK option does not disable the error that occurs, be- + cause it applies only to the testing of input strings for UTF validity. If the extra option PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES is set, surro- gate code point values in UTF-8 and UTF-32 patterns no longer provoke @@ -1865,13 +1919,13 @@ COMPILING A PATTERN If the PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL extra option is passed to pcre2_compile(), all unrecognized or malformed escape sequences are treated as single-character escapes. For example, \j is a literal "j" - and \x{2z} is treated as the literal string "x{2z}". Setting this - option means that typos in patterns may go undetected and have unex- - pected results. Also note that a sequence such as [\N{] is interpreted - as a malformed attempt at [\N{...}] and so is treated as [N{] whereas - [\N] gives an error because an unqualified \N is a valid escape - sequence but is not supported in a character class. To reiterate: this - is a dangerous option. Use with great care. + and \x{2z} is treated as the literal string "x{2z}". Setting this op- + tion means that typos in patterns may go undetected and have unexpected + results. Also note that a sequence such as [\N{] is interpreted as a + malformed attempt at [\N{...}] and so is treated as [N{] whereas [\N] + gives an error because an unqualified \N is a valid escape sequence but + is not supported in a character class. To reiterate: this is a danger- + ous option. Use with great care. PCRE2_EXTRA_ESCAPED_CR_IS_LF @@ -1888,8 +1942,8 @@ COMPILING A PATTERN causes the pattern only to match complete lines. This is achieved by automatically inserting the code for "^(?:" at the start of the com- piled pattern and ")$" at the end. Thus, when PCRE2_MULTILINE is set, - the matched line may be in the middle of the subject string. This - option can be used with PCRE2_LITERAL. + the matched line may be in the middle of the subject string. This op- + tion can be used with PCRE2_LITERAL. PCRE2_EXTRA_MATCH_WORD @@ -1935,39 +1989,52 @@ JUST-IN-TIME (JIT) COMPILATION LOCALE SUPPORT + const uint8_t *pcre2_maketables(pcre2_general_context *gcontext); + + void pcre2_maketables_free(pcre2_general_context *gcontext, + const uint8_t *tables); + PCRE2 handles caseless matching, and determines whether characters are letters, digits, or whatever, by reference to a set of tables, indexed - by character code point. This applies only to characters whose code - points are less than 256. By default, higher-valued code points never - match escapes such as \w or \d. However, if PCRE2 is built with Uni- - code support, all characters can be tested with \p and \P, or, alterna- + by character code point. However, this applies only to characters whose + code points are less than 256. By default, higher-valued code points + never match escapes such as \w or \d. + + When PCRE2 is built with Unicode support (the default), the Unicode + properties of all characters can be tested with \p and \P, or, alterna- tively, the PCRE2_UCP option can be set when a pattern is compiled; this causes \w and friends to use Unicode property support instead of - the built-in tables. + the built-in tables. PCRE2_UCP also causes upper/lower casing opera- + tions on characters with code points greater than 127 to use Unicode + properties. These effects apply even when PCRE2_UTF is not set. The use of locales with Unicode is discouraged. If you are handling - characters with code points greater than 128, you should either use + characters with code points greater than 127, you should either use Unicode support, or use locales, but not try to mix the two. - PCRE2 contains an internal set of character tables that are used by - default. These are sufficient for many applications. Normally, the - internal tables recognize only ASCII characters. However, when PCRE2 is + PCRE2 contains a built-in set of character tables that are used by de- + fault. These are sufficient for many applications. Normally, the in- + ternal tables recognize only ASCII characters. However, when PCRE2 is built, it is possible to cause the internal tables to be rebuilt in the default "C" locale of the local system, which may cause them to be dif- ferent. - The internal tables can be overridden by tables supplied by the appli- + The built-in tables can be overridden by tables supplied by the appli- cation that calls PCRE2. These may be created in a different locale from the default. As more and more applications change to using Uni- code, the need for this locale support is expected to die away. External tables are built by calling the pcre2_maketables() function, - in the relevant locale. The result can be passed to pcre2_compile() as - often as necessary, by creating a compile context and calling - pcre2_set_character_tables() to set the tables pointer therein. For - example, to build and use tables that are appropriate for the French - locale (where accented characters with values greater than 128 are - treated as letters), the following code could be used: + in the relevant locale. The only argument to this function is a general + context, which can be used to pass a custom memory allocator. If the + argument is NULL, the system malloc() is used. The result can be passed + to pcre2_compile() as often as necessary, by creating a compile context + and calling pcre2_set_character_tables() to set the tables pointer + therein. + + For example, to build and use tables that are appropriate for the + French locale (where accented characters with values greater than 127 + are treated as letters), the following code could be used: setlocale(LC_CTYPE, "fr_FR"); tables = pcre2_maketables(NULL); @@ -1976,15 +2043,33 @@ LOCALE SUPPORT re = pcre2_compile(..., ccontext); The locale name "fr_FR" is used on Linux and other Unix-like systems; - if you are using Windows, the name for the French locale is "french". - It is the caller's responsibility to ensure that the memory containing - the tables remains available for as long as it is needed. + if you are using Windows, the name for the French locale is "french". The pointer that is passed (via the compile context) to pcre2_compile() - is saved with the compiled pattern, and the same tables are used by - pcre2_match() and pcre_dfa_match(). Thus, for any single pattern, com- - pilation and matching both happen in the same locale, but different - patterns can be processed in different locales. + is saved with the compiled pattern, and the same tables are used by the + matching functions. Thus, for any single pattern, compilation and + matching both happen in the same locale, but different patterns can be + processed in different locales. + + It is the caller's responsibility to ensure that the memory containing + the tables remains available while they are still in use. When they are + no longer needed, you can discard them using pcre2_maketables_free(), + which should pass as its first parameter the same global context that + was used to create the tables. + + Saving locale tables + + The tables described above are just a sequence of binary bytes, which + makes them independent of hardware characteristics such as endianness + or whether the processor is 32-bit or 64-bit. A copy of the result of + pcre2_maketables() can therefore be saved in a file or elsewhere and + re-used later, even in a different program or on another computer. The + size of the tables (number of bytes) must be obtained by calling + pcre2_config() with the PCRE2_CONFIG_TABLES_LENGTH option because + pcre2_maketables() does not return this value. Note that the + pcre2_dftables program, which is part of the PCRE2 build system, can be + used stand-alone to create a file that contains a set of binary tables. + See the pcre2build documentation for details. INFORMATION ABOUT A COMPILED PATTERN @@ -1995,8 +2080,8 @@ INFORMATION ABOUT A COMPILED PATTERN compiled pattern. For information about callouts, see the next section. The first argument for pcre2_pattern_info() is a pointer to the com- piled pattern. The second argument specifies which piece of information - is required, and the third argument is a pointer to a variable to - receive the data. If the third argument is NULL, the first argument is + is required, and the third argument is a pointer to a variable to re- + ceive the data. If the third argument is NULL, the first argument is ignored, and the function returns the size in bytes of the variable that is required for the information requested. Otherwise, the yield of the function is zero for success, or one of the following negative num- @@ -2007,8 +2092,8 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_ERROR_BADOPTION the value of what was invalid PCRE2_ERROR_UNSET the requested field is not set - The "magic number" is placed at the start of each compiled pattern as - an simple check against passing an arbitrary memory pointer. Here is a + The "magic number" is placed at the start of each compiled pattern as a + simple check against passing an arbitrary memory pointer. Here is a typical call of pcre2_pattern_info(), to obtain the length of the com- piled pattern: @@ -2027,20 +2112,20 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_EXTRAOPTIONS Return copies of the pattern's options. The third argument should point - to a uint32_t variable. PCRE2_INFO_ARGOPTIONS returns exactly the - options that were passed to pcre2_compile(), whereas PCRE2_INFO_ALLOP- + to a uint32_t variable. PCRE2_INFO_ARGOPTIONS returns exactly the op- + tions that were passed to pcre2_compile(), whereas PCRE2_INFO_ALLOP- TIONS returns the compile options as modified by any top-level (*XXX) option settings such as (*UTF) at the start of the pattern itself. PCRE2_INFO_EXTRAOPTIONS returns the extra options that were set in the compile context by calling the pcre2_set_compile_extra_options() func- tion. - For example, if the pattern /(*UTF)abc/ is compiled with the - PCRE2_EXTENDED option, the result for PCRE2_INFO_ALLOPTIONS is - PCRE2_EXTENDED and PCRE2_UTF. Option settings such as (?i) that can - change within a pattern do not affect the result of PCRE2_INFO_ALLOP- - TIONS, even if they appear right at the start of the pattern. (This was - different in some earlier releases.) + For example, if the pattern /(*UTF)abc/ is compiled with the PCRE2_EX- + TENDED option, the result for PCRE2_INFO_ALLOPTIONS is PCRE2_EXTENDED + and PCRE2_UTF. Option settings such as (?i) that can change within a + pattern do not affect the result of PCRE2_INFO_ALLOPTIONS, even if they + appear right at the start of the pattern. (This was different in some + earlier releases.) A pattern compiled without PCRE2_ANCHORED is automatically anchored by PCRE2 if the first significant item in every top-level branch is one of @@ -2067,7 +2152,7 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_BACKREFMAX Return the number of the highest backreference in the pattern. The - third argument should point to an uint32_t variable. Named capture + third argument should point to a uint32_t variable. Named capture groups acquire numbers as well as names, and these count towards the highest backreference. Backreferences such as \4 or \g{12} match the captured characters of the given group, but in addition, the check that @@ -2085,17 +2170,17 @@ INFORMATION ABOUT A COMPILED PATTERN Return the highest capture group number in the pattern. In patterns where (?| is not used, this is also the total number of capture groups. - The third argument should point to an uint32_t variable. + The third argument should point to a uint32_t variable. PCRE2_INFO_DEPTHLIMIT If the pattern set a backtracking depth limit by including an item of the form (*LIMIT_DEPTH=nnnn) at the start, the value is returned. The third argument should point to a uint32_t integer. If no such value has - been set, the call to pcre2_pattern_info() returns the error - PCRE2_ERROR_UNSET. Note that this limit will only be used during match- - ing if it is less than the limit set or defaulted by the caller of the - match function. + been set, the call to pcre2_pattern_info() returns the error PCRE2_ER- + ROR_UNSET. Note that this limit will only be used during matching if it + is less than the limit set or defaulted by the caller of the match + function. PCRE2_INFO_FIRSTBITMAP @@ -2111,7 +2196,7 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_FIRSTCODETYPE Return information about the first code unit of any matched string, for - a non-anchored pattern. The third argument should point to an uint32_t + a non-anchored pattern. The third argument should point to a uint32_t variable. If there is a fixed first value, for example, the letter "c" from a pattern such as (cat|cow|coyote), 1 is returned, and the value can be retrieved using PCRE2_INFO_FIRSTCODEUNIT. If there is no fixed @@ -2123,7 +2208,7 @@ INFORMATION ABOUT A COMPILED PATTERN Return the value of the first code unit of any matched string for a pattern where PCRE2_INFO_FIRSTCODETYPE returns 1; otherwise return 0. - The third argument should point to an uint32_t variable. In the 8-bit + The third argument should point to a uint32_t variable. In the 8-bit library, the value is always less than 256. In the 16-bit library the value can be up to 0xffff. In the 32-bit library in UTF-32 mode the value can be up to 0x10ffff, and up to 0xffffffff when not using UTF-32 @@ -2141,15 +2226,15 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_HASBACKSLASHC Return 1 if the pattern contains any instances of \C, otherwise 0. The - third argument should point to an uint32_t variable. + third argument should point to a uint32_t variable. PCRE2_INFO_HASCRORLF Return 1 if the pattern contains any explicit matches for CR or LF - characters, otherwise 0. The third argument should point to an uint32_t + characters, otherwise 0. The third argument should point to a uint32_t variable. An explicit match is either a literal CR or LF character, or - \r or \n or one of the equivalent hexadecimal or octal escape - sequences. + \r or \n or one of the equivalent hexadecimal or octal escape se- + quences. PCRE2_INFO_HEAPLIMIT @@ -2163,7 +2248,7 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_JCHANGED Return 1 if the (?J) or (?-J) option setting is used in the pattern, - otherwise 0. The third argument should point to an uint32_t variable. + otherwise 0. The third argument should point to a uint32_t variable. (?J) and (?-J) set and unset the local PCRE2_DUPNAMES option, respec- tively. @@ -2177,112 +2262,121 @@ INFORMATION ABOUT A COMPILED PATTERN Returns 1 if there is a rightmost literal code unit that must exist in any matched string, other than at its start. The third argument should - point to an uint32_t variable. If there is no such value, 0 is - returned. When 1 is returned, the code unit value itself can be - retrieved using PCRE2_INFO_LASTCODEUNIT. For anchored patterns, a last - literal value is recorded only if it follows something of variable - length. For example, for the pattern /^a\d+z\d+/ the returned value is - 1 (with "z" returned from PCRE2_INFO_LASTCODEUNIT), but for /^a\dz\d/ - the returned value is 0. + point to a uint32_t variable. If there is no such value, 0 is returned. + When 1 is returned, the code unit value itself can be retrieved using + PCRE2_INFO_LASTCODEUNIT. For anchored patterns, a last literal value is + recorded only if it follows something of variable length. For example, + for the pattern /^a\d+z\d+/ the returned value is 1 (with "z" returned + from PCRE2_INFO_LASTCODEUNIT), but for /^a\dz\d/ the returned value is + 0. PCRE2_INFO_LASTCODEUNIT Return the value of the rightmost literal code unit that must exist in any matched string, other than at its start, for a pattern where PCRE2_INFO_LASTCODETYPE returns 1. Otherwise, return 0. The third argu- - ment should point to an uint32_t variable. + ment should point to a uint32_t variable. PCRE2_INFO_MATCHEMPTY Return 1 if the pattern might match an empty string, otherwise 0. The - third argument should point to an uint32_t variable. When a pattern - contains recursive subroutine calls it is not always possible to deter- - mine whether or not it can match an empty string. PCRE2 takes a cau- - tious approach and returns 1 in such cases. + third argument should point to a uint32_t variable. When a pattern con- + tains recursive subroutine calls it is not always possible to determine + whether or not it can match an empty string. PCRE2 takes a cautious ap- + proach and returns 1 in such cases. PCRE2_INFO_MATCHLIMIT If the pattern set a match limit by including an item of the form - (*LIMIT_MATCH=nnnn) at the start, the value is returned. The third - argument should point to a uint32_t integer. If no such value has been - set, the call to pcre2_pattern_info() returns the error - PCRE2_ERROR_UNSET. Note that this limit will only be used during match- - ing if it is less than the limit set or defaulted by the caller of the - match function. + (*LIMIT_MATCH=nnnn) at the start, the value is returned. The third ar- + gument should point to a uint32_t integer. If no such value has been + set, the call to pcre2_pattern_info() returns the error PCRE2_ERROR_UN- + SET. Note that this limit will only be used during matching if it is + less than the limit set or defaulted by the caller of the match func- + tion. PCRE2_INFO_MAXLOOKBEHIND - Return the number of characters (not code units) in the longest lookbe- - hind assertion in the pattern. The third argument should point to a - uint32_t integer. This information is useful when doing multi-segment - matching using the partial matching facilities. Note that the simple - assertions \b and \B require a one-character lookbehind. \A also regis- - ters a one-character lookbehind, though it does not actually inspect - the previous character. This is to ensure that at least one character - from the old segment is retained when a new segment is processed. Oth- - erwise, if there are no lookbehinds in the pattern, \A might match - incorrectly at the start of a second or subsequent segment. + A lookbehind assertion moves back a certain number of characters (not + code units) when it starts to process each of its branches. This re- + quest returns the largest of these backward moves. The third argument + should point to a uint32_t integer. The simple assertions \b and \B re- + quire a one-character lookbehind and cause PCRE2_INFO_MAXLOOKBEHIND to + return 1 in the absence of anything longer. \A also registers a one- + character lookbehind, though it does not actually inspect the previous + character. + + Note that this information is useful for multi-segment matching only if + the pattern contains no nested lookbehinds. For example, the pattern + (?<=a(?<=ba)c) returns a maximum lookbehind of 2, but when it is pro- + cessed, the first lookbehind moves back by two characters, matches one + character, then the nested lookbehind also moves back by two charac- + ters. This puts the matching point three characters earlier than it was + at the start. PCRE2_INFO_MAXLOOKBEHIND is really only useful as a de- + bugging tool. See the pcre2partial documentation for a discussion of + multi-segment matching. PCRE2_INFO_MINLENGTH If a minimum length for matching subject strings was computed, its - value is returned. Otherwise the returned value is 0. The value is a - number of characters, which in UTF mode may be different from the num- - ber of code units. The third argument should point to an uint32_t - variable. The value is a lower bound to the length of any matching - string. There may not be any strings of that length that do actually - match, but every string that does match is at least that long. + value is returned. Otherwise the returned value is 0. This value is not + computed when PCRE2_NO_START_OPTIMIZE is set. The value is a number of + characters, which in UTF mode may be different from the number of code + units. The third argument should point to a uint32_t variable. The + value is a lower bound to the length of any matching string. There may + not be any strings of that length that do actually match, but every + string that does match is at least that long. PCRE2_INFO_NAMECOUNT PCRE2_INFO_NAMEENTRYSIZE PCRE2_INFO_NAMETABLE PCRE2 supports the use of named as well as numbered capturing parenthe- - ses. The names are just an additional way of identifying the parenthe- + ses. The names are just an additional way of identifying the parenthe- ses, which still acquire numbers. Several convenience functions such as - pcre2_substring_get_byname() are provided for extracting captured sub- - strings by name. It is also possible to extract the data directly, by - first converting the name to a number in order to access the correct - pointers in the output vector (described with pcre2_match() below). To - do the conversion, you need to use the name-to-number map, which is - described by these three values. - - The map consists of a number of fixed-size entries. PCRE2_INFO_NAME- - COUNT gives the number of entries, and PCRE2_INFO_NAMEENTRYSIZE gives - the size of each entry in code units; both of these return a uint32_t + pcre2_substring_get_byname() are provided for extracting captured sub- + strings by name. It is also possible to extract the data directly, by + first converting the name to a number in order to access the correct + pointers in the output vector (described with pcre2_match() below). To + do the conversion, you need to use the name-to-number map, which is de- + scribed by these three values. + + The map consists of a number of fixed-size entries. PCRE2_INFO_NAME- + COUNT gives the number of entries, and PCRE2_INFO_NAMEENTRYSIZE gives + the size of each entry in code units; both of these return a uint32_t value. The entry size depends on the length of the longest name. PCRE2_INFO_NAMETABLE returns a pointer to the first entry of the table. - This is a PCRE2_SPTR pointer to a block of code units. In the 8-bit - library, the first two bytes of each entry are the number of the cap- - turing parenthesis, most significant byte first. In the 16-bit library, - the pointer points to 16-bit code units, the first of which contains - the parenthesis number. In the 32-bit library, the pointer points to - 32-bit code units, the first of which contains the parenthesis number. + This is a PCRE2_SPTR pointer to a block of code units. In the 8-bit li- + brary, the first two bytes of each entry are the number of the captur- + ing parenthesis, most significant byte first. In the 16-bit library, + the pointer points to 16-bit code units, the first of which contains + the parenthesis number. In the 32-bit library, the pointer points to + 32-bit code units, the first of which contains the parenthesis number. The rest of the entry is the corresponding name, zero terminated. - The names are in alphabetical order. If (?| is used to create multiple - capture groups with the same number, as described in the section on - duplicate group numbers in the pcre2pattern page, the groups may be - given the same name, but there is only one entry in the table. Differ- - ent names for groups of the same number are not permitted. + The names are in alphabetical order. If (?| is used to create multiple + capture groups with the same number, as described in the section on du- + plicate group numbers in the pcre2pattern page, the groups may be given + the same name, but there is only one entry in the table. Different + names for groups of the same number are not permitted. - Duplicate names for capture groups with different numbers are permit- + Duplicate names for capture groups with different numbers are permit- ted, but only if PCRE2_DUPNAMES is set. They appear in the table in the - order in which they were found in the pattern. In the absence of (?| - this is the order of increasing number; when (?| is used this is not - necessarily the case because later capture groups may have lower num- + order in which they were found in the pattern. In the absence of (?| + this is the order of increasing number; when (?| is used this is not + necessarily the case because later capture groups may have lower num- bers. - As a simple example of the name/number table, consider the following - pattern after compilation by the 8-bit library (assume PCRE2_EXTENDED + As a simple example of the name/number table, consider the following + pattern after compilation by the 8-bit library (assume PCRE2_EXTENDED is set, so white space - including newlines - is ignored): (?<date> (?<year>(\d\d)?\d\d) - (?<month>\d\d) - (?<day>\d\d) ) There are four named capture groups, so the table has four entries, and - each entry in the table is eight bytes long. The table is as follows, + each entry in the table is eight bytes long. The table is as follows, with non-printing bytes shows in hexadecimal, and undefined bytes shown as ??: @@ -2291,8 +2385,8 @@ INFORMATION ABOUT A COMPILED PATTERN 00 04 m o n t h 00 00 02 y e a r 00 ?? - When writing code to extract data from named capture groups using the - name-to-number map, remember that the length of the entries is likely + When writing code to extract data from named capture groups using the + name-to-number map, remember that the length of the entries is likely to be different for each compiled pattern. PCRE2_INFO_NEWLINE @@ -2311,15 +2405,15 @@ INFORMATION ABOUT A COMPILED PATTERN PCRE2_INFO_SIZE - Return the size of the compiled pattern in bytes (for all three - libraries). The third argument should point to a size_t variable. This - value includes the size of the general data block that precedes the - code units of the compiled pattern itself. The value that is used when - pcre2_compile() is getting memory in which to place the compiled pat- - tern may be slightly larger than the value returned by this option, - because there are cases where the code that calculates the size has to - over-estimate. Processing a pattern with the JIT compiler does not - alter the value returned by this option. + Return the size of the compiled pattern in bytes (for all three li- + braries). The third argument should point to a size_t variable. This + value includes the size of the general data block that precedes the + code units of the compiled pattern itself. The value that is used when + pcre2_compile() is getting memory in which to place the compiled pat- + tern may be slightly larger than the value returned by this option, be- + cause there are cases where the code that calculates the size has to + over-estimate. Processing a pattern with the JIT compiler does not al- + ter the value returned by this option. INFORMATION ABOUT A PATTERN'S CALLOUTS @@ -2329,30 +2423,30 @@ INFORMATION ABOUT A PATTERN'S CALLOUTS void *user_data); A script language that supports the use of string arguments in callouts - might like to scan all the callouts in a pattern before running the + might like to scan all the callouts in a pattern before running the match. This can be done by calling pcre2_callout_enumerate(). The first - argument is a pointer to a compiled pattern, the second points to a - callback function, and the third is arbitrary user data. The callback - function is called for every callout in the pattern in the order in + argument is a pointer to a compiled pattern, the second points to a + callback function, and the third is arbitrary user data. The callback + function is called for every callout in the pattern in the order in which they appear. Its first argument is a pointer to a callout enumer- - ation block, and its second argument is the user_data value that was - passed to pcre2_callout_enumerate(). The contents of the callout enu- - meration block are described in the pcre2callout documentation, which + ation block, and its second argument is the user_data value that was + passed to pcre2_callout_enumerate(). The contents of the callout enu- + meration block are described in the pcre2callout documentation, which also gives further details about callouts. SERIALIZATION AND PRECOMPILING - It is possible to save compiled patterns on disc or elsewhere, and - reload them later, subject to a number of restrictions. The host on - which the patterns are reloaded must be running the same version of + It is possible to save compiled patterns on disc or elsewhere, and + reload them later, subject to a number of restrictions. The host on + which the patterns are reloaded must be running the same version of PCRE2, with the same code unit width, and must also have the same endi- - anness, pointer width, and PCRE2_SIZE type. Before compiled patterns - can be saved, they must be converted to a "serialized" form, which in - the case of PCRE2 is really just a bytecode dump. The functions whose - names begin with pcre2_serialize_ are used for converting to and from - the serialized form. They are described in the pcre2serialize documen- - tation. Note that PCRE2 serialization does not convert compiled pat- + anness, pointer width, and PCRE2_SIZE type. Before compiled patterns + can be saved, they must be converted to a "serialized" form, which in + the case of PCRE2 is really just a bytecode dump. The functions whose + names begin with pcre2_serialize_ are used for converting to and from + the serialized form. They are described in the pcre2serialize documen- + tation. Note that PCRE2 serialization does not convert compiled pat- terns to an abstract format like Java or .NET serialization. @@ -2366,60 +2460,58 @@ THE MATCH DATA BLOCK void pcre2_match_data_free(pcre2_match_data *match_data); - Information about a successful or unsuccessful match is placed in a - match data block, which is an opaque structure that is accessed by - function calls. In particular, the match data block contains a vector - of offsets into the subject string that define the matched part of the - subject and any substrings that were captured. This is known as the + Information about a successful or unsuccessful match is placed in a + match data block, which is an opaque structure that is accessed by + function calls. In particular, the match data block contains a vector + of offsets into the subject string that define the matched part of the + subject and any substrings that were captured. This is known as the ovector. - Before calling pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match() + Before calling pcre2_match(), pcre2_dfa_match(), or pcre2_jit_match() you must create a match data block by calling one of the creation func- - tions above. For pcre2_match_data_create(), the first argument is the - number of pairs of offsets in the ovector. One pair of offsets is - required to identify the string that matched the whole pattern, with an - additional pair for each captured substring. For example, a value of 4 - creates enough space to record the matched portion of the subject plus - three captured substrings. A minimum of at least 1 pair is imposed by + tions above. For pcre2_match_data_create(), the first argument is the + number of pairs of offsets in the ovector. One pair of offsets is re- + quired to identify the string that matched the whole pattern, with an + additional pair for each captured substring. For example, a value of 4 + creates enough space to record the matched portion of the subject plus + three captured substrings. A minimum of at least 1 pair is imposed by pcre2_match_data_create(), so it is always possible to return the over- all matched string. The second argument of pcre2_match_data_create() is a pointer to a gen- - eral context, which can specify custom memory management for obtaining + eral context, which can specify custom memory management for obtaining the memory for the match data block. If you are not using custom memory management, pass NULL, which causes malloc() to be used. - For pcre2_match_data_create_from_pattern(), the first argument is a + For pcre2_match_data_create_from_pattern(), the first argument is a pointer to a compiled pattern. The ovector is created to be exactly the right size to hold all the substrings a pattern might capture. The sec- - ond argument is again a pointer to a general context, but in this case + ond argument is again a pointer to a general context, but in this case if NULL is passed, the memory is obtained using the same allocator that was used for the compiled pattern (custom or default). - A match data block can be used many times, with the same or different - compiled patterns. You can extract information from a match data block - after a match operation has finished, using functions that are - described in the sections on matched strings and other match data - below. + A match data block can be used many times, with the same or different + compiled patterns. You can extract information from a match data block + after a match operation has finished, using functions that are de- + scribed in the sections on matched strings and other match data below. When a call of pcre2_match() fails, valid data is available in the - match block only when the error is PCRE2_ERROR_NOMATCH, - PCRE2_ERROR_PARTIAL, or one of the error codes for an invalid UTF - string. Exactly what is available depends on the error, and is detailed - below. - - When one of the matching functions is called, pointers to the compiled - pattern and the subject string are set in the match data block so that - they can be referenced by the extraction functions after a successful + match block only when the error is PCRE2_ERROR_NOMATCH, PCRE2_ER- + ROR_PARTIAL, or one of the error codes for an invalid UTF string. Ex- + actly what is available depends on the error, and is detailed below. + + When one of the matching functions is called, pointers to the compiled + pattern and the subject string are set in the match data block so that + they can be referenced by the extraction functions after a successful match. After running a match, you must not free a compiled pattern or a - subject string until after all operations on the match data block (for - that match) have taken place, unless, in the case of the subject - string, you have used the PCRE2_COPY_MATCHED_SUBJECT option, which is - described in the section entitled "Option bits for pcre2_match()" - below. - - When a match data block itself is no longer needed, it should be freed - by calling pcre2_match_data_free(). If this function is called with a + subject string until after all operations on the match data block (for + that match) have taken place, unless, in the case of the subject + string, you have used the PCRE2_COPY_MATCHED_SUBJECT option, which is + described in the section entitled "Option bits for pcre2_match()" be- + low. + + When a match data block itself is no longer needed, it should be freed + by calling pcre2_match_data_free(). If this function is called with a NULL argument, it returns immediately, without doing anything. @@ -2430,15 +2522,15 @@ MATCHING A PATTERN: THE TRADITIONAL FUNCTION uint32_t options, pcre2_match_data *match_data, pcre2_match_context *mcontext); - The function pcre2_match() is called to match a subject string against - a compiled pattern, which is passed in the code argument. You can call + The function pcre2_match() is called to match a subject string against + a compiled pattern, which is passed in the code argument. You can call pcre2_match() with the same code argument as many times as you like, in - order to find multiple matches in the subject string or to match dif- + order to find multiple matches in the subject string or to match dif- ferent subject strings with the same pattern. - This function is the main matching facility of the library, and it - operates in a Perl-like manner. For specialist use there is also an - alternative matching function, which is described below in the section + This function is the main matching facility of the library, and it op- + erates in a Perl-like manner. For specialist use there is also an al- + ternative matching function, which is described below in the section about the pcre2_dfa_match() function. Here is an example of a simple call to pcre2_match(): @@ -2453,205 +2545,215 @@ MATCHING A PATTERN: THE TRADITIONAL FUNCTION md, /* the match data block */ NULL); /* a match context; NULL means use defaults */ - If the subject string is zero-terminated, the length can be given as + If the subject string is zero-terminated, the length can be given as PCRE2_ZERO_TERMINATED. A match context must be provided if certain less common matching parameters are to be changed. For details, see the sec- tion on the match context above. The string to be matched by pcre2_match() - The subject string is passed to pcre2_match() as a pointer in subject, - a length in length, and a starting offset in startoffset. The length - and offset are in code units, not characters. That is, they are in - bytes for the 8-bit library, 16-bit code units for the 16-bit library, - and 32-bit code units for the 32-bit library, whether or not UTF pro- + The subject string is passed to pcre2_match() as a pointer in subject, + a length in length, and a starting offset in startoffset. The length + and offset are in code units, not characters. That is, they are in + bytes for the 8-bit library, 16-bit code units for the 16-bit library, + and 32-bit code units for the 32-bit library, whether or not UTF pro- cessing is enabled. If startoffset is greater than the length of the subject, pcre2_match() - returns PCRE2_ERROR_BADOFFSET. When the starting offset is zero, the - search for a match starts at the beginning of the subject, and this is + returns PCRE2_ERROR_BADOFFSET. When the starting offset is zero, the + search for a match starts at the beginning of the subject, and this is by far the most common case. In UTF-8 or UTF-16 mode, the starting off- - set must point to the start of a character, or to the end of the sub- - ject (in UTF-32 mode, one code unit equals one character, so all off- - sets are valid). Like the pattern string, the subject may contain - binary zeros. - - A non-zero starting offset is useful when searching for another match - in the same subject by calling pcre2_match() again after a previous - success. Setting startoffset differs from passing over a shortened - string and setting PCRE2_NOTBOL in the case of a pattern that begins + set must point to the start of a character, or to the end of the sub- + ject (in UTF-32 mode, one code unit equals one character, so all off- + sets are valid). Like the pattern string, the subject may contain bi- + nary zeros. + + A non-zero starting offset is useful when searching for another match + in the same subject by calling pcre2_match() again after a previous + success. Setting startoffset differs from passing over a shortened + string and setting PCRE2_NOTBOL in the case of a pattern that begins with any kind of lookbehind. For example, consider the pattern \Biss\B - which finds occurrences of "iss" in the middle of words. (\B matches - only if the current position in the subject is not a word boundary.) + which finds occurrences of "iss" in the middle of words. (\B matches + only if the current position in the subject is not a word boundary.) When applied to the string "Mississipi" the first call to pcre2_match() - finds the first occurrence. If pcre2_match() is called again with just - the remainder of the subject, namely "issipi", it does not match, - because \B is always false at the start of the subject, which is deemed - to be a word boundary. However, if pcre2_match() is passed the entire + finds the first occurrence. If pcre2_match() is called again with just + the remainder of the subject, namely "issipi", it does not match, be- + cause \B is always false at the start of the subject, which is deemed + to be a word boundary. However, if pcre2_match() is passed the entire string again, but with startoffset set to 4, it finds the second occur- - rence of "iss" because it is able to look behind the starting point to + rence of "iss" because it is able to look behind the starting point to discover that it is preceded by a letter. - Finding all the matches in a subject is tricky when the pattern can + Finding all the matches in a subject is tricky when the pattern can match an empty string. It is possible to emulate Perl's /g behaviour by - first trying the match again at the same offset, with the - PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED options, and then if that - fails, advancing the starting offset and trying an ordinary match - again. There is some code that demonstrates how to do this in the - pcre2demo sample program. In the most general case, you have to check - to see if the newline convention recognizes CRLF as a newline, and if - so, and the current character is CR followed by LF, advance the start- + first trying the match again at the same offset, with the + PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED options, and then if that + fails, advancing the starting offset and trying an ordinary match + again. There is some code that demonstrates how to do this in the + pcre2demo sample program. In the most general case, you have to check + to see if the newline convention recognizes CRLF as a newline, and if + so, and the current character is CR followed by LF, advance the start- ing offset by two characters instead of one. If a non-zero starting offset is passed when the pattern is anchored, a single attempt to match at the given offset is made. This can only suc- - ceed if the pattern does not require the match to be at the start of - the subject. In other words, the anchoring must be the result of set- - ting the PCRE2_ANCHORED option or the use of .* with PCRE2_DOTALL, not + ceed if the pattern does not require the match to be at the start of + the subject. In other words, the anchoring must be the result of set- + ting the PCRE2_ANCHORED option or the use of .* with PCRE2_DOTALL, not by starting the pattern with ^ or \A. Option bits for pcre2_match() The unused bits of the options argument for pcre2_match() must be zero. - The only bits that may be set are PCRE2_ANCHORED, - PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, - PCRE2_NOTEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, PCRE2_NO_JIT, - PCRE2_NO_UTF_CHECK, PCRE2_PARTIAL_HARD, and PCRE2_PARTIAL_SOFT. Their + The only bits that may be set are PCRE2_ANCHORED, + PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, PCRE2_NO- + TEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, PCRE2_NO_JIT, + PCRE2_NO_UTF_CHECK, PCRE2_PARTIAL_HARD, and PCRE2_PARTIAL_SOFT. Their action is described below. - Setting PCRE2_ANCHORED or PCRE2_ENDANCHORED at match time is not sup- - ported by the just-in-time (JIT) compiler. If it is set, JIT matching - is disabled and the interpretive code in pcre2_match() is run. Apart - from PCRE2_NO_JIT (obviously), the remaining options are supported for + Setting PCRE2_ANCHORED or PCRE2_ENDANCHORED at match time is not sup- + ported by the just-in-time (JIT) compiler. If it is set, JIT matching + is disabled and the interpretive code in pcre2_match() is run. Apart + from PCRE2_NO_JIT (obviously), the remaining options are supported for JIT matching. PCRE2_ANCHORED The PCRE2_ANCHORED option limits pcre2_match() to matching at the first - matching position. If a pattern was compiled with PCRE2_ANCHORED, or - turned out to be anchored by virtue of its contents, it cannot be made - unachored at matching time. Note that setting the option at match time + matching position. If a pattern was compiled with PCRE2_ANCHORED, or + turned out to be anchored by virtue of its contents, it cannot be made + unachored at matching time. Note that setting the option at match time disables JIT matching. PCRE2_COPY_MATCHED_SUBJECT - By default, a pointer to the subject is remembered in the match data - block so that, after a successful match, it can be referenced by the - substring extraction functions. This means that the subject's memory - must not be freed until all such operations are complete. For some - applications where the lifetime of the subject string is not guaran- - teed, it may be necessary to make a copy of the subject string, but it - is wasteful to do this unless the match is successful. After a success- - ful match, if PCRE2_COPY_MATCHED_SUBJECT is set, the subject is copied - and the new pointer is remembered in the match data block instead of - the original subject pointer. The memory allocator that was used for - the match block itself is used. The copy is automatically freed when - pcre2_match_data_free() is called to free the match data block. It is + By default, a pointer to the subject is remembered in the match data + block so that, after a successful match, it can be referenced by the + substring extraction functions. This means that the subject's memory + must not be freed until all such operations are complete. For some ap- + plications where the lifetime of the subject string is not guaranteed, + it may be necessary to make a copy of the subject string, but it is + wasteful to do this unless the match is successful. After a successful + match, if PCRE2_COPY_MATCHED_SUBJECT is set, the subject is copied and + the new pointer is remembered in the match data block instead of the + original subject pointer. The memory allocator that was used for the + match block itself is used. The copy is automatically freed when + pcre2_match_data_free() is called to free the match data block. It is also automatically freed if the match data block is re-used for another match operation. PCRE2_ENDANCHORED - If the PCRE2_ENDANCHORED option is set, any string that pcre2_match() - matches must be right at the end of the subject string. Note that set- + If the PCRE2_ENDANCHORED option is set, any string that pcre2_match() + matches must be right at the end of the subject string. Note that set- ting the option at match time disables JIT matching. PCRE2_NOTBOL This option specifies that first character of the subject string is not - the beginning of a line, so the circumflex metacharacter should not - match before it. Setting this without having set PCRE2_MULTILINE at + the beginning of a line, so the circumflex metacharacter should not + match before it. Setting this without having set PCRE2_MULTILINE at compile time causes circumflex never to match. This option affects only the behaviour of the circumflex metacharacter. It does not affect \A. PCRE2_NOTEOL This option specifies that the end of the subject string is not the end - of a line, so the dollar metacharacter should not match it nor (except - in multiline mode) a newline immediately before it. Setting this with- - out having set PCRE2_MULTILINE at compile time causes dollar never to + of a line, so the dollar metacharacter should not match it nor (except + in multiline mode) a newline immediately before it. Setting this with- + out having set PCRE2_MULTILINE at compile time causes dollar never to match. This option affects only the behaviour of the dollar metacharac- ter. It does not affect \Z or \z. PCRE2_NOTEMPTY An empty string is not considered to be a valid match if this option is - set. If there are alternatives in the pattern, they are tried. If all - the alternatives match the empty string, the entire match fails. For + set. If there are alternatives in the pattern, they are tried. If all + the alternatives match the empty string, the entire match fails. For example, if the pattern a?b? - is applied to a string not beginning with "a" or "b", it matches an + is applied to a string not beginning with "a" or "b", it matches an empty string at the start of the subject. With PCRE2_NOTEMPTY set, this - match is not valid, so pcre2_match() searches further into the string + match is not valid, so pcre2_match() searches further into the string for occurrences of "a" or "b". PCRE2_NOTEMPTY_ATSTART - This is like PCRE2_NOTEMPTY, except that it locks out an empty string + This is like PCRE2_NOTEMPTY, except that it locks out an empty string match only at the first matching position, that is, at the start of the - subject plus the starting offset. An empty string match later in the - subject is permitted. If the pattern is anchored, such a match can - occur only if the pattern contains \K. + subject plus the starting offset. An empty string match later in the + subject is permitted. If the pattern is anchored, such a match can oc- + cur only if the pattern contains \K. PCRE2_NO_JIT - By default, if a pattern has been successfully processed by - pcre2_jit_compile(), JIT is automatically used when pcre2_match() is - called with options that JIT supports. Setting PCRE2_NO_JIT disables + By default, if a pattern has been successfully processed by + pcre2_jit_compile(), JIT is automatically used when pcre2_match() is + called with options that JIT supports. Setting PCRE2_NO_JIT disables the use of JIT; it forces matching to be done by the interpreter. PCRE2_NO_UTF_CHECK When PCRE2_UTF is set at compile time, the validity of the subject as a - UTF string is checked by default when pcre2_match() is subsequently - called. If a non-zero starting offset is given, the check is applied - only to that part of the subject that could be inspected during match- - ing, and there is a check that the starting offset points to the first - code unit of a character or to the end of the subject. If there are no - lookbehind assertions in the pattern, the check starts at the starting - offset. Otherwise, it starts at the length of the longest lookbehind - before the starting offset, or at the start of the subject if there are - not that many characters before the starting offset. Note that the - sequences \b and \B are one-character lookbehinds. + UTF string is checked unless PCRE2_NO_UTF_CHECK is passed to + pcre2_match() or PCRE2_MATCH_INVALID_UTF was passed to pcre2_compile(). + The latter special case is discussed in detail in the pcre2unicode doc- + umentation. + + In the default case, if a non-zero starting offset is given, the check + is applied only to that part of the subject that could be inspected + during matching, and there is a check that the starting offset points + to the first code unit of a character or to the end of the subject. If + there are no lookbehind assertions in the pattern, the check starts at + the starting offset. Otherwise, it starts at the length of the longest + lookbehind before the starting offset, or at the start of the subject + if there are not that many characters before the starting offset. Note + that the sequences \b and \B are one-character lookbehinds. The check is carried out before any other processing takes place, and a - negative error code is returned if the check fails. There are several - UTF error codes for each code unit width, corresponding to different - problems with the code unit sequence. There are discussions about the - validity of UTF-8 strings, UTF-16 strings, and UTF-32 strings in the - pcre2unicode page. - - If you know that your subject is valid, and you want to skip these - checks for performance reasons, you can set the PCRE2_NO_UTF_CHECK - option when calling pcre2_match(). You might want to do this for the - second and subsequent calls to pcre2_match() if you are making repeated - calls to find other matches in the same subject string. - - Warning: When PCRE2_NO_UTF_CHECK is set, the effect of passing an - invalid string as a subject, or an invalid value of startoffset, is - undefined. Your program may crash or loop indefinitely. + negative error code is returned if the check fails. There are several + UTF error codes for each code unit width, corresponding to different + problems with the code unit sequence. There are discussions about the + validity of UTF-8 strings, UTF-16 strings, and UTF-32 strings in the + pcre2unicode documentation. + + If you know that your subject is valid, and you want to skip this check + for performance reasons, you can set the PCRE2_NO_UTF_CHECK option when + calling pcre2_match(). You might want to do this for the second and + subsequent calls to pcre2_match() if you are making repeated calls to + find multiple matches in the same subject string. + + Warning: Unless PCRE2_MATCH_INVALID_UTF was set at compile time, when + PCRE2_NO_UTF_CHECK is set at match time the effect of passing an in- + valid string as a subject, or an invalid value of startoffset, is unde- + fined. Your program may crash or loop indefinitely or give wrong re- + sults. PCRE2_PARTIAL_HARD PCRE2_PARTIAL_SOFT - These options turn on the partial matching feature. A partial match - occurs if the end of the subject string is reached successfully, but - there are not enough subject characters to complete the match. If this - happens when PCRE2_PARTIAL_SOFT (but not PCRE2_PARTIAL_HARD) is set, - matching continues by testing any remaining alternatives. Only if no - complete match can be found is PCRE2_ERROR_PARTIAL returned instead of - PCRE2_ERROR_NOMATCH. In other words, PCRE2_PARTIAL_SOFT specifies that - the caller is prepared to handle a partial match, but only if no com- - plete match can be found. - - If PCRE2_PARTIAL_HARD is set, it overrides PCRE2_PARTIAL_SOFT. In this - case, if a partial match is found, pcre2_match() immediately returns - PCRE2_ERROR_PARTIAL, without considering any other alternatives. In + These options turn on the partial matching feature. A partial match oc- + curs if the end of the subject string is reached successfully, but + there are not enough subject characters to complete the match. In addi- + tion, either at least one character must have been inspected or the + pattern must contain a lookbehind, or the pattern must be one that + could match an empty string. + + If this situation arises when PCRE2_PARTIAL_SOFT (but not PCRE2_PAR- + TIAL_HARD) is set, matching continues by testing any remaining alterna- + tives. Only if no complete match can be found is PCRE2_ERROR_PARTIAL + returned instead of PCRE2_ERROR_NOMATCH. In other words, PCRE2_PAR- + TIAL_SOFT specifies that the caller is prepared to handle a partial + match, but only if no complete match can be found. + + If PCRE2_PARTIAL_HARD is set, it overrides PCRE2_PARTIAL_SOFT. In this + case, if a partial match is found, pcre2_match() immediately returns + PCRE2_ERROR_PARTIAL, without considering any other alternatives. In other words, when PCRE2_PARTIAL_HARD is set, a partial match is consid- ered to be more important that an alternative complete match. @@ -2661,38 +2763,38 @@ MATCHING A PATTERN: THE TRADITIONAL FUNCTION NEWLINE HANDLING WHEN MATCHING - When PCRE2 is built, a default newline convention is set; this is usu- - ally the standard convention for the operating system. The default can - be overridden in a compile context by calling pcre2_set_newline(). It - can also be overridden by starting a pattern string with, for example, - (*CRLF), as described in the section on newline conventions in the - pcre2pattern page. During matching, the newline choice affects the be- - haviour of the dot, circumflex, and dollar metacharacters. It may also - alter the way the match starting position is advanced after a match + When PCRE2 is built, a default newline convention is set; this is usu- + ally the standard convention for the operating system. The default can + be overridden in a compile context by calling pcre2_set_newline(). It + can also be overridden by starting a pattern string with, for example, + (*CRLF), as described in the section on newline conventions in the + pcre2pattern page. During matching, the newline choice affects the be- + haviour of the dot, circumflex, and dollar metacharacters. It may also + alter the way the match starting position is advanced after a match failure for an unanchored pattern. When PCRE2_NEWLINE_CRLF, PCRE2_NEWLINE_ANYCRLF, or PCRE2_NEWLINE_ANY is - set as the newline convention, and a match attempt for an unanchored + set as the newline convention, and a match attempt for an unanchored pattern fails when the current starting position is at a CRLF sequence, - and the pattern contains no explicit matches for CR or LF characters, - the match position is advanced by two characters instead of one, in + and the pattern contains no explicit matches for CR or LF characters, + the match position is advanced by two characters instead of one, in other words, to after the CRLF. The above rule is a compromise that makes the most common cases work as - expected. For example, if the pattern is .+A (and the PCRE2_DOTALL - option is not set), it does not match the string "\r\nA" because, after - failing at the start, it skips both the CR and the LF before retrying. - However, the pattern [\r\n]A does match that string, because it con- + expected. For example, if the pattern is .+A (and the PCRE2_DOTALL op- + tion is not set), it does not match the string "\r\nA" because, after + failing at the start, it skips both the CR and the LF before retrying. + However, the pattern [\r\n]A does match that string, because it con- tains an explicit CR or LF reference, and so advances only by one char- acter after the first failure. An explicit match for CR of LF is either a literal appearance of one of - those characters in the pattern, or one of the \r or \n or equivalent + those characters in the pattern, or one of the \r or \n or equivalent octal or hexadecimal escape sequences. Implicit matches such as [^X] do - not count, nor does \s, even though it includes CR and LF in the char- + not count, nor does \s, even though it includes CR and LF in the char- acters that it matches. - Notwithstanding the above, anomalous effects may still occur when CRLF + Notwithstanding the above, anomalous effects may still occur when CRLF is a valid newline sequence and explicit \r or \n escapes appear in the pattern. @@ -2703,82 +2805,82 @@ HOW PCRE2_MATCH() RETURNS A STRING AND CAPTURED SUBSTRINGS PCRE2_SIZE *pcre2_get_ovector_pointer(pcre2_match_data *match_data); - In general, a pattern matches a certain portion of the subject, and in - addition, further substrings from the subject may be picked out by - parenthesized parts of the pattern. Following the usage in Jeffrey - Friedl's book, this is called "capturing" in what follows, and the - phrase "capture group" (Perl terminology) is used for a fragment of a - pattern that picks out a substring. PCRE2 supports several other kinds + In general, a pattern matches a certain portion of the subject, and in + addition, further substrings from the subject may be picked out by + parenthesized parts of the pattern. Following the usage in Jeffrey + Friedl's book, this is called "capturing" in what follows, and the + phrase "capture group" (Perl terminology) is used for a fragment of a + pattern that picks out a substring. PCRE2 supports several other kinds of parenthesized group that do not cause substrings to be captured. The - pcre2_pattern_info() function can be used to find out how many capture + pcre2_pattern_info() function can be used to find out how many capture groups there are in a compiled pattern. - You can use auxiliary functions for accessing captured substrings by + You can use auxiliary functions for accessing captured substrings by number or by name, as described in sections below. Alternatively, you can make direct use of the vector of PCRE2_SIZE val- - ues, called the ovector, which contains the offsets of captured - strings. It is part of the match data block. The function - pcre2_get_ovector_pointer() returns the address of the ovector, and + ues, called the ovector, which contains the offsets of captured + strings. It is part of the match data block. The function + pcre2_get_ovector_pointer() returns the address of the ovector, and pcre2_get_ovector_count() returns the number of pairs of values it con- tains. Within the ovector, the first in each pair of values is set to the off- set of the first code unit of a substring, and the second is set to the - offset of the first code unit after the end of a substring. These val- - ues are always code unit offsets, not character offsets. That is, they - are byte offsets in the 8-bit library, 16-bit offsets in the 16-bit - library, and 32-bit offsets in the 32-bit library. - - After a partial match (error return PCRE2_ERROR_PARTIAL), only the - first pair of offsets (that is, ovector[0] and ovector[1]) are set. - They identify the part of the subject that was partially matched. See + offset of the first code unit after the end of a substring. These val- + ues are always code unit offsets, not character offsets. That is, they + are byte offsets in the 8-bit library, 16-bit offsets in the 16-bit li- + brary, and 32-bit offsets in the 32-bit library. + + After a partial match (error return PCRE2_ERROR_PARTIAL), only the + first pair of offsets (that is, ovector[0] and ovector[1]) are set. + They identify the part of the subject that was partially matched. See the pcre2partial documentation for details of partial matching. - After a fully successful match, the first pair of offsets identifies - the portion of the subject string that was matched by the entire pat- - tern. The next pair is used for the first captured substring, and so - on. The value returned by pcre2_match() is one more than the highest - numbered pair that has been set. For example, if two substrings have - been captured, the returned value is 3. If there are no captured sub- + After a fully successful match, the first pair of offsets identifies + the portion of the subject string that was matched by the entire pat- + tern. The next pair is used for the first captured substring, and so + on. The value returned by pcre2_match() is one more than the highest + numbered pair that has been set. For example, if two substrings have + been captured, the returned value is 3. If there are no captured sub- strings, the return value from a successful match is 1, indicating that just the first pair of offsets has been set. - If a pattern uses the \K escape sequence within a positive assertion, + If a pattern uses the \K escape sequence within a positive assertion, the reported start of a successful match can be greater than the end of - the match. For example, if the pattern (?=ab\K) is matched against + the match. For example, if the pattern (?=ab\K) is matched against "ab", the start and end offset values for the match are 2 and 0. - If a capture group is matched repeatedly within a single match opera- - tion, it is the last portion of the subject that it matched that is - returned. + If a capture group is matched repeatedly within a single match opera- + tion, it is the last portion of the subject that it matched that is re- + turned. If the ovector is too small to hold all the captured substring offsets, - as much as possible is filled in, and the function returns a value of - zero. If captured substrings are not of interest, pcre2_match() may be + as much as possible is filled in, and the function returns a value of + zero. If captured substrings are not of interest, pcre2_match() may be called with a match data block whose ovector is of minimum length (that is, one pair). - It is possible for capture group number n+1 to match some part of the - subject when group n has not been used at all. For example, if the + It is possible for capture group number n+1 to match some part of the + subject when group n has not been used at all. For example, if the string "abc" is matched against the pattern (a|(z))(bc) the return from - the function is 4, and groups 1 and 3 are matched, but 2 is not. When - this happens, both values in the offset pairs corresponding to unused + the function is 4, and groups 1 and 3 are matched, but 2 is not. When + this happens, both values in the offset pairs corresponding to unused groups are set to PCRE2_UNSET. - Offset values that correspond to unused groups at the end of the - expression are also set to PCRE2_UNSET. For example, if the string - "abc" is matched against the pattern (abc)(x(yz)?)? groups 2 and 3 are - not matched. The return from the function is 2, because the highest - used capture group number is 1. The offsets for for the second and - third capture groupss (assuming the vector is large enough, of course) - are set to PCRE2_UNSET. + Offset values that correspond to unused groups at the end of the ex- + pression are also set to PCRE2_UNSET. For example, if the string "abc" + is matched against the pattern (abc)(x(yz)?)? groups 2 and 3 are not + matched. The return from the function is 2, because the highest used + capture group number is 1. The offsets for for the second and third + capture groupss (assuming the vector is large enough, of course) are + set to PCRE2_UNSET. Elements in the ovector that do not correspond to capturing parentheses in the pattern are never changed. That is, if a pattern contains n cap- turing parentheses, no more than ovector[0] to ovector[2n+1] are set by - pcre2_match(). The other elements retain whatever values they previ- - ously had. After a failed match attempt, the contents of the ovector + pcre2_match(). The other elements retain whatever values they previ- + ously had. After a failed match attempt, the contents of the ovector are unchanged. @@ -2788,69 +2890,69 @@ OTHER INFORMATION ABOUT A MATCH PCRE2_SIZE pcre2_get_startchar(pcre2_match_data *match_data); - As well as the offsets in the ovector, other information about a match - is retained in the match data block and can be retrieved by the above - functions in appropriate circumstances. If they are called at other + As well as the offsets in the ovector, other information about a match + is retained in the match data block and can be retrieved by the above + functions in appropriate circumstances. If they are called at other times, the result is undefined. - After a successful match, a partial match (PCRE2_ERROR_PARTIAL), or a - failure to match (PCRE2_ERROR_NOMATCH), a mark name may be available. - The function pcre2_get_mark() can be called to access this name, which - can be specified in the pattern by any of the backtracking control + After a successful match, a partial match (PCRE2_ERROR_PARTIAL), or a + failure to match (PCRE2_ERROR_NOMATCH), a mark name may be available. + The function pcre2_get_mark() can be called to access this name, which + can be specified in the pattern by any of the backtracking control verbs, not just (*MARK). The same function applies to all the verbs. It returns a pointer to the zero-terminated name, which is within the com- piled pattern. If no name is available, NULL is returned. The length of - the name (excluding the terminating zero) is stored in the code unit - that precedes the name. You should use this length instead of relying + the name (excluding the terminating zero) is stored in the code unit + that precedes the name. You should use this length instead of relying on the terminating zero if the name might contain a binary zero. - After a successful match, the name that is returned is the last mark + After a successful match, the name that is returned is the last mark name encountered on the matching path through the pattern. Instances of - backtracking verbs without names do not count. Thus, for example, if + backtracking verbs without names do not count. Thus, for example, if the matching path contains (*MARK:A)(*PRUNE), the name "A" is returned. - After a "no match" or a partial match, the last encountered name is - returned. For example, consider this pattern: + After a "no match" or a partial match, the last encountered name is re- + turned. For example, consider this pattern: ^(*MARK:A)((*MARK:B)a|b)c - When it matches "bc", the returned name is A. The B mark is "seen" in - the first branch of the group, but it is not on the matching path. On - the other hand, when this pattern fails to match "bx", the returned + When it matches "bc", the returned name is A. The B mark is "seen" in + the first branch of the group, but it is not on the matching path. On + the other hand, when this pattern fails to match "bx", the returned name is B. - Warning: By default, certain start-of-match optimizations are used to - give a fast "no match" result in some situations. For example, if the - anchoring is removed from the pattern above, there is an initial check - for the presence of "c" in the subject before running the matching - engine. This check fails for "bx", causing a match failure without see- - ing any marks. You can disable the start-of-match optimizations by set- - ting the PCRE2_NO_START_OPTIMIZE option for pcre2_compile() or by - starting the pattern with (*NO_START_OPT). - - After a successful match, a partial match, or one of the invalid UTF - errors (for example, PCRE2_ERROR_UTF8_ERR5), pcre2_get_startchar() can + Warning: By default, certain start-of-match optimizations are used to + give a fast "no match" result in some situations. For example, if the + anchoring is removed from the pattern above, there is an initial check + for the presence of "c" in the subject before running the matching en- + gine. This check fails for "bx", causing a match failure without seeing + any marks. You can disable the start-of-match optimizations by setting + the PCRE2_NO_START_OPTIMIZE option for pcre2_compile() or by starting + the pattern with (*NO_START_OPT). + + After a successful match, a partial match, or one of the invalid UTF + errors (for example, PCRE2_ERROR_UTF8_ERR5), pcre2_get_startchar() can be called. After a successful or partial match it returns the code unit - offset of the character at which the match started. For a non-partial - match, this can be different to the value of ovector[0] if the pattern - contains the \K escape sequence. After a partial match, however, this - value is always the same as ovector[0] because \K does not affect the + offset of the character at which the match started. For a non-partial + match, this can be different to the value of ovector[0] if the pattern + contains the \K escape sequence. After a partial match, however, this + value is always the same as ovector[0] because \K does not affect the result of a partial match. - After a UTF check failure, pcre2_get_startchar() can be used to obtain + After a UTF check failure, pcre2_get_startchar() can be used to obtain the code unit offset of the invalid UTF character. Details are given in the pcre2unicode page. ERROR RETURNS FROM pcre2_match() - If pcre2_match() fails, it returns a negative number. This can be con- - verted to a text string by calling the pcre2_get_error_message() func- - tion (see "Obtaining a textual error message" below). Negative error - codes are also returned by other functions, and are documented with - them. The codes are given names in the header file. If UTF checking is + If pcre2_match() fails, it returns a negative number. This can be con- + verted to a text string by calling the pcre2_get_error_message() func- + tion (see "Obtaining a textual error message" below). Negative error + codes are also returned by other functions, and are documented with + them. The codes are given names in the header file. If UTF checking is in force and an invalid UTF subject string is detected, one of a number - of UTF-specific negative error codes is returned. Details are given in - the pcre2unicode page. The following are the other errors that may be + of UTF-specific negative error codes is returned. Details are given in + the pcre2unicode page. The following are the other errors that may be returned by pcre2_match(): PCRE2_ERROR_NOMATCH @@ -2859,20 +2961,20 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_PARTIAL - The subject string did not match, but it did match partially. See the + The subject string did not match, but it did match partially. See the pcre2partial documentation for details of partial matching. PCRE2_ERROR_BADMAGIC PCRE2 stores a 4-byte "magic number" at the start of the compiled code, - to catch the case when it is passed a junk pointer. This is the error + to catch the case when it is passed a junk pointer. This is the error that is returned when the magic number is not present. PCRE2_ERROR_BADMODE - This error is given when a compiled pattern is passed to a function in - a library of a different code unit width, for example, a pattern com- - piled by the 8-bit library is passed to a 16-bit or 32-bit library + This error is given when a compiled pattern is passed to a function in + a library of a different code unit width, for example, a pattern com- + piled by the 8-bit library is passed to a 16-bit or 32-bit library function. PCRE2_ERROR_BADOFFSET @@ -2886,15 +2988,15 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_BADUTFOFFSET The UTF code unit sequence that was passed as a subject was checked and - found to be valid (the PCRE2_NO_UTF_CHECK option was not set), but the - value of startoffset did not point to the beginning of a UTF character + found to be valid (the PCRE2_NO_UTF_CHECK option was not set), but the + value of startoffset did not point to the beginning of a UTF character or the end of the subject. PCRE2_ERROR_CALLOUT - This error is never generated by pcre2_match() itself. It is provided - for use by callout functions that want to cause pcre2_match() or - pcre2_callout_enumerate() to return a distinctive error code. See the + This error is never generated by pcre2_match() itself. It is provided + for use by callout functions that want to cause pcre2_match() or + pcre2_callout_enumerate() to return a distinctive error code. See the pcre2callout documentation for details. PCRE2_ERROR_DEPTHLIMIT @@ -2907,15 +3009,15 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_INTERNAL - An unexpected internal error has occurred. This error could be caused + An unexpected internal error has occurred. This error could be caused by a bug in PCRE2 or by overwriting of the compiled pattern. PCRE2_ERROR_JIT_STACKLIMIT - This error is returned when a pattern that was successfully studied - using JIT is being matched, but the memory available for the just-in- - time processing stack is not large enough. See the pcre2jit documenta- - tion for more details. + This error is returned when a pattern that was successfully studied us- + ing JIT is being matched, but the memory available for the just-in-time + processing stack is not large enough. See the pcre2jit documentation + for more details. PCRE2_ERROR_MATCHLIMIT @@ -2923,11 +3025,11 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_NOMEMORY - If a pattern contains many nested backtracking points, heap memory is - used to remember them. This error is given when the memory allocation - function (default or custom) fails. Note that a different error, - PCRE2_ERROR_HEAPLIMIT, is given if the amount of memory needed exceeds - the heap limit. PCRE2_ERROR_NOMEMORY is also returned if + If a pattern contains many nested backtracking points, heap memory is + used to remember them. This error is given when the memory allocation + function (default or custom) fails. Note that a different error, + PCRE2_ERROR_HEAPLIMIT, is given if the amount of memory needed exceeds + the heap limit. PCRE2_ERROR_NOMEMORY is also returned if PCRE2_COPY_MATCHED_SUBJECT is set and memory allocation fails. PCRE2_ERROR_NULL @@ -2936,12 +3038,12 @@ ERROR RETURNS FROM pcre2_match() PCRE2_ERROR_RECURSELOOP - This error is returned when pcre2_match() detects a recursion loop - within the pattern. Specifically, it means that either the whole pat- + This error is returned when pcre2_match() detects a recursion loop + within the pattern. Specifically, it means that either the whole pat- tern or a capture group has been called recursively for the second time - at the same position in the subject string. Some simple patterns that - might do this are detected and faulted at compile time, but more com- - plicated cases, in particular mutual recursions between two different + at the same position in the subject string. Some simple patterns that + might do this are detected and faulted at compile time, but more com- + plicated cases, in particular mutual recursions between two different groups, cannot be detected until matching is attempted. @@ -2950,21 +3052,21 @@ OBTAINING A TEXTUAL ERROR MESSAGE int pcre2_get_error_message(int errorcode, PCRE2_UCHAR *buffer, PCRE2_SIZE bufflen); - A text message for an error code from any PCRE2 function (compile, - match, or auxiliary) can be obtained by calling pcre2_get_error_mes- - sage(). The code is passed as the first argument, with the remaining - two arguments specifying a code unit buffer and its length in code - units, into which the text message is placed. The message is returned - in code units of the appropriate width for the library that is being + A text message for an error code from any PCRE2 function (compile, + match, or auxiliary) can be obtained by calling pcre2_get_error_mes- + sage(). The code is passed as the first argument, with the remaining + two arguments specifying a code unit buffer and its length in code + units, into which the text message is placed. The message is returned + in code units of the appropriate width for the library that is being used. - The returned message is terminated with a trailing zero, and the func- - tion returns the number of code units used, excluding the trailing - zero. If the error number is unknown, the negative error code - PCRE2_ERROR_BADDATA is returned. If the buffer is too small, the mes- - sage is truncated (but still with a trailing zero), and the negative - error code PCRE2_ERROR_NOMEMORY is returned. None of the messages are - very long; a buffer size of 120 code units is ample. + The returned message is terminated with a trailing zero, and the func- + tion returns the number of code units used, excluding the trailing + zero. If the error number is unknown, the negative error code PCRE2_ER- + ROR_BADDATA is returned. If the buffer is too small, the message is + truncated (but still with a trailing zero), and the negative error code + PCRE2_ERROR_NOMEMORY is returned. None of the messages are very long; + a buffer size of 120 code units is ample. EXTRACTING CAPTURED SUBSTRINGS BY NUMBER @@ -2982,39 +3084,39 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER void pcre2_substring_free(PCRE2_UCHAR *buffer); - Captured substrings can be accessed directly by using the ovector as + Captured substrings can be accessed directly by using the ovector as described above. For convenience, auxiliary functions are provided for - extracting captured substrings as new, separate, zero-terminated + extracting captured substrings as new, separate, zero-terminated strings. A substring that contains a binary zero is correctly extracted - and has a further zero added on the end, but the result is not, of + and has a further zero added on the end, but the result is not, of course, a C string. The functions in this section identify substrings by number. The number zero refers to the entire matched substring, with higher numbers refer- - ring to substrings captured by parenthesized groups. After a partial - match, only substring zero is available. An attempt to extract any - other substring gives the error PCRE2_ERROR_PARTIAL. The next section + ring to substrings captured by parenthesized groups. After a partial + match, only substring zero is available. An attempt to extract any + other substring gives the error PCRE2_ERROR_PARTIAL. The next section describes similar functions for extracting captured substrings by name. - If a pattern uses the \K escape sequence within a positive assertion, + If a pattern uses the \K escape sequence within a positive assertion, the reported start of a successful match can be greater than the end of - the match. For example, if the pattern (?=ab\K) is matched against - "ab", the start and end offset values for the match are 2 and 0. In - this situation, calling these functions with a zero substring number + the match. For example, if the pattern (?=ab\K) is matched against + "ab", the start and end offset values for the match are 2 and 0. In + this situation, calling these functions with a zero substring number extracts a zero-length empty string. - You can find the length in code units of a captured substring without - extracting it by calling pcre2_substring_length_bynumber(). The first - argument is a pointer to the match data block, the second is the group - number, and the third is a pointer to a variable into which the length - is placed. If you just want to know whether or not the substring has + You can find the length in code units of a captured substring without + extracting it by calling pcre2_substring_length_bynumber(). The first + argument is a pointer to the match data block, the second is the group + number, and the third is a pointer to a variable into which the length + is placed. If you just want to know whether or not the substring has been captured, you can pass the third argument as NULL. - The pcre2_substring_copy_bynumber() function copies a captured sub- - string into a supplied buffer, whereas pcre2_substring_get_bynumber() - copies it into new memory, obtained using the same memory allocation - function that was used for the match data block. The first two argu- - ments of these functions are a pointer to the match data block and a + The pcre2_substring_copy_bynumber() function copies a captured sub- + string into a supplied buffer, whereas pcre2_substring_get_bynumber() + copies it into new memory, obtained using the same memory allocation + function that was used for the match data block. The first two argu- + ments of these functions are a pointer to the match data block and a capture group number. The final arguments of pcre2_substring_copy_bynumber() are a pointer to @@ -3023,25 +3125,25 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER for the extracted substring, excluding the terminating zero. For pcre2_substring_get_bynumber() the third and fourth arguments point - to variables that are updated with a pointer to the new memory and the - number of code units that comprise the substring, again excluding the - terminating zero. When the substring is no longer needed, the memory + to variables that are updated with a pointer to the new memory and the + number of code units that comprise the substring, again excluding the + terminating zero. When the substring is no longer needed, the memory should be freed by calling pcre2_substring_free(). - The return value from all these functions is zero for success, or a - negative error code. If the pattern match failed, the match failure - code is returned. If a substring number greater than zero is used - after a partial match, PCRE2_ERROR_PARTIAL is returned. Other possible + The return value from all these functions is zero for success, or a + negative error code. If the pattern match failed, the match failure + code is returned. If a substring number greater than zero is used af- + ter a partial match, PCRE2_ERROR_PARTIAL is returned. Other possible error codes are: PCRE2_ERROR_NOMEMORY - The buffer was too small for pcre2_substring_copy_bynumber(), or the + The buffer was too small for pcre2_substring_copy_bynumber(), or the attempt to get memory failed for pcre2_substring_get_bynumber(). PCRE2_ERROR_NOSUBSTRING - There is no substring with that number in the pattern, that is, the + There is no substring with that number in the pattern, that is, the number is greater than the number of capturing parentheses. PCRE2_ERROR_UNAVAILABLE @@ -3052,8 +3154,8 @@ EXTRACTING CAPTURED SUBSTRINGS BY NUMBER PCRE2_ERROR_UNSET - The substring did not participate in the match. For example, if the - pattern is (abc)|(def) and the subject is "def", and the ovector con- + The substring did not participate in the match. For example, if the + pattern is (abc)|(def) and the subject is "def", and the ovector con- tains at least two capturing slots, substring number 1 is unset. @@ -3064,31 +3166,31 @@ EXTRACTING A LIST OF ALL CAPTURED SUBSTRINGS void pcre2_substring_list_free(PCRE2_SPTR *list); - The pcre2_substring_list_get() function extracts all available sub- - strings and builds a list of pointers to them. It also (optionally) - builds a second list that contains their lengths (in code units), - excluding a terminating zero that is added to each of them. All this is + The pcre2_substring_list_get() function extracts all available sub- + strings and builds a list of pointers to them. It also (optionally) + builds a second list that contains their lengths (in code units), ex- + cluding a terminating zero that is added to each of them. All this is done in a single block of memory that is obtained using the same memory allocation function that was used to get the match data block. - This function must be called only after a successful match. If called + This function must be called only after a successful match. If called after a partial match, the error code PCRE2_ERROR_PARTIAL is returned. - The address of the memory block is returned via listptr, which is also + The address of the memory block is returned via listptr, which is also the start of the list of string pointers. The end of the list is marked - by a NULL pointer. The address of the list of lengths is returned via - lengthsptr. If your strings do not contain binary zeros and you do not + by a NULL pointer. The address of the list of lengths is returned via + lengthsptr. If your strings do not contain binary zeros and you do not therefore need the lengths, you may supply NULL as the lengthsptr argu- - ment to disable the creation of a list of lengths. The yield of the - function is zero if all went well, or PCRE2_ERROR_NOMEMORY if the mem- - ory block could not be obtained. When the list is no longer needed, it + ment to disable the creation of a list of lengths. The yield of the + function is zero if all went well, or PCRE2_ERROR_NOMEMORY if the mem- + ory block could not be obtained. When the list is no longer needed, it should be freed by calling pcre2_substring_list_free(). If this function encounters a substring that is unset, which can happen - when capture group number n+1 matches some part of the subject, but - group n has not been used at all, it returns an empty string. This can + when capture group number n+1 matches some part of the subject, but + group n has not been used at all, it returns an empty string. This can be distinguished from a genuine zero-length substring by inspecting the - appropriate offset in the ovector, which contain PCRE2_UNSET for unset + appropriate offset in the ovector, which contain PCRE2_UNSET for unset substrings, or by calling pcre2_substring_length_bynumber(). @@ -3108,7 +3210,7 @@ EXTRACTING CAPTURED SUBSTRINGS BY NAME void pcre2_substring_free(PCRE2_UCHAR *buffer); - To extract a substring by name, you first have to find associated num- + To extract a substring by name, you first have to find associated num- ber. For example, for this pattern: (a+)b(?<xxx>\d+)... @@ -3116,32 +3218,32 @@ EXTRACTING CAPTURED SUBSTRINGS BY NAME the number of the capture group called "xxx" is 2. If the name is known to be unique (PCRE2_DUPNAMES was not set), you can find the number from the name by calling pcre2_substring_number_from_name(). The first argu- - ment is the compiled pattern, and the second is the name. The yield of - the function is the group number, PCRE2_ERROR_NOSUBSTRING if there is - no group with that name, or PCRE2_ERROR_NOUNIQUESUBSTRING if there is - more than one group with that name. Given the number, you can extract - the substring directly from the ovector, or use one of the "bynumber" + ment is the compiled pattern, and the second is the name. The yield of + the function is the group number, PCRE2_ERROR_NOSUBSTRING if there is + no group with that name, or PCRE2_ERROR_NOUNIQUESUBSTRING if there is + more than one group with that name. Given the number, you can extract + the substring directly from the ovector, or use one of the "bynumber" functions described above. - For convenience, there are also "byname" functions that correspond to - the "bynumber" functions, the only difference being that the second - argument is a name instead of a number. If PCRE2_DUPNAMES is set and + For convenience, there are also "byname" functions that correspond to + the "bynumber" functions, the only difference being that the second ar- + gument is a name instead of a number. If PCRE2_DUPNAMES is set and there are duplicate names, these functions scan all the groups with the - given name, and return the captured substring from the first named + given name, and return the captured substring from the first named group that is set. - If there are no groups with the given name, PCRE2_ERROR_NOSUBSTRING is - returned. If all groups with the name have numbers that are greater - than the number of slots in the ovector, PCRE2_ERROR_UNAVAILABLE is - returned. If there is at least one group with a slot in the ovector, - but no group is found to be set, PCRE2_ERROR_UNSET is returned. + If there are no groups with the given name, PCRE2_ERROR_NOSUBSTRING is + returned. If all groups with the name have numbers that are greater + than the number of slots in the ovector, PCRE2_ERROR_UNAVAILABLE is re- + turned. If there is at least one group with a slot in the ovector, but + no group is found to be set, PCRE2_ERROR_UNSET is returned. Warning: If the pattern uses the (?| feature to set up multiple capture - groups with the same number, as described in the section on duplicate + groups with the same number, as described in the section on duplicate group numbers in the pcre2pattern page, you cannot use names to distin- - guish the different capture groups, because names are not included in - the compiled code. The matching process uses only numbers. For this - reason, the use of different names for groups with the same number + guish the different capture groups, because names are not included in + the compiled code. The matching process uses only numbers. For this + reason, the use of different names for groups with the same number causes an error at compile time. @@ -3154,13 +3256,20 @@ CREATING A NEW STRING WITH SUBSTITUTIONS PCRE2_SIZE rlength, PCRE2_UCHAR *outputbuffer, PCRE2_SIZE *outlengthptr); - This function calls pcre2_match() and then makes a copy of the subject - string in outputbuffer, replacing one or more parts that were matched + This function optionally calls pcre2_match() and then makes a copy of + the subject string in outputbuffer, replacing parts that were matched with the replacement string, whose length is supplied in rlength. This - can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. - The default is to perform just one replacement, but there is an option - that requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL below - for details). + can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. + There is an option (see PCRE2_SUBSTITUTE_REPLACEMENT_ONLY below) to re- + turn just the replacement string(s). The default action is to perform + just one replacement if the pattern matches, but there is an option + that requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL be- + low). + + If successful, pcre2_substitute() returns the number of substitutions + that were carried out. This may be zero if no match was found, and is + never greater than one unless PCRE2_SUBSTITUTE_GLOBAL is set. A nega- + tive value is returned if an error is detected. Matches in which a \K item in a lookahead in the pattern causes the match to end before it starts are not supported, and give rise to an @@ -3175,60 +3284,104 @@ CREATING A NEW STRING WITH SUBSTITUTIONS ment functions from the match context, if provided, or else those that were used to allocate memory for the compiled code. - If an external match_data block is provided, its contents afterwards - are those set by the final call to pcre2_match(). For global changes, - this will have ended in a matching error. The contents of the ovector - within the match data block may or may not have been changed. - - The outlengthptr argument must point to a variable that contains the - length, in code units, of the output buffer. If the function is suc- - cessful, the value is updated to contain the length of the new string, - excluding the trailing zero that is automatically added. - - If the function is not successful, the value set via outlengthptr - depends on the type of error. For syntax errors in the replacement - string, the value is the offset in the replacement string where the - error was detected. For other errors, the value is PCRE2_UNSET by - default. This includes the case of the output buffer being too small, - unless PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set (see below), in which - case the value is the minimum length needed, including space for the - trailing zero. Note that in order to compute the required length, - pcre2_substitute() has to simulate all the matching and copying, - instead of giving an error return as soon as the buffer overflows. Note - also that the length is in code units, not bytes. - - In the replacement string, which is interpreted as a UTF string in UTF - mode, and is checked for UTF validity unless the PCRE2_NO_UTF_CHECK - option is set, a dollar character is an escape character that can spec- - ify the insertion of characters from capture groups or names from - (*MARK) or other control verbs in the pattern. The following forms are - always recognized: + If match_data is not NULL and PCRE2_SUBSTITUTE_MATCHED is not set, the + provided block is used for all calls to pcre2_match(), and its contents + afterwards are the result of the final call. For global changes, this + will always be a no-match error. The contents of the ovector within the + match data block may or may not have been changed. + + As well as the usual options for pcre2_match(), a number of additional + options can be set in the options argument of pcre2_substitute(). One + such option is PCRE2_SUBSTITUTE_MATCHED. When this is set, an external + match_data block must be provided, and it must have been used for an + external call to pcre2_match(). The data in the match_data block (re- + turn code, offset vector) is used for the first substitution instead of + calling pcre2_match() from within pcre2_substitute(). This allows an + application to check for a match before choosing to substitute, without + having to repeat the match. + + The contents of the externally supplied match data block are not + changed when PCRE2_SUBSTITUTE_MATCHED is set. If PCRE2_SUBSTI- + TUTE_GLOBAL is also set, pcre2_match() is called after the first sub- + stitution to check for further matches, but this is done using an in- + ternally obtained match data block, thus always leaving the external + block unchanged. + + The code argument is not used for matching before the first substitu- + tion when PCRE2_SUBSTITUTE_MATCHED is set, but it must be provided, + even when PCRE2_SUBSTITUTE_GLOBAL is not set, because it contains in- + formation such as the UTF setting and the number of capturing parenthe- + ses in the pattern. + + The default action of pcre2_substitute() is to return a copy of the + subject string with matched substrings replaced. However, if PCRE2_SUB- + STITUTE_REPLACEMENT_ONLY is set, only the replacement substrings are + returned. In the global case, multiple replacements are concatenated in + the output buffer. Substitution callouts (see below) can be used to + separate them if necessary. + + The outlengthptr argument of pcre2_substitute() must point to a vari- + able that contains the length, in code units, of the output buffer. If + the function is successful, the value is updated to contain the length + in code units of the new string, excluding the trailing zero that is + automatically added. + + If the function is not successful, the value set via outlengthptr de- + pends on the type of error. For syntax errors in the replacement + string, the value is the offset in the replacement string where the er- + ror was detected. For other errors, the value is PCRE2_UNSET by de- + fault. This includes the case of the output buffer being too small, un- + less PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set. + + PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output + buffer is too small. The default action is to return PCRE2_ERROR_NOMEM- + ORY immediately. If this option is set, however, pcre2_substitute() + continues to go through the motions of matching and substituting (with- + out, of course, writing anything) in order to compute the size of buf- + fer that is needed. This value is passed back via the outlengthptr + variable, with the result of the function still being PCRE2_ER- + ROR_NOMEMORY. + + Passing a buffer size of zero is a permitted way of finding out how + much memory is needed for given substitution. However, this does mean + that the entire operation is carried out twice. Depending on the appli- + cation, it may be more efficient to allocate a large buffer and free + the excess afterwards, instead of using PCRE2_SUBSTITUTE_OVER- + FLOW_LENGTH. + + The replacement string, which is interpreted as a UTF string in UTF + mode, is checked for UTF validity unless PCRE2_NO_UTF_CHECK is set. An + invalid UTF replacement string causes an immediate return with the rel- + evant UTF error code. + + If PCRE2_SUBSTITUTE_LITERAL is set, the replacement string is not in- + terpreted in any way. By default, however, a dollar character is an es- + cape character that can specify the insertion of characters from cap- + ture groups and names from (*MARK) or other control verbs in the pat- + tern. The following forms are always recognized: $$ insert a dollar character $<n> or ${<n>} insert the contents of group <n> $*MARK or ${*MARK} insert a control verb name - Either a group number or a group name can be given for <n>. Curly - brackets are required only if the following character would be inter- + Either a group number or a group name can be given for <n>. Curly + brackets are required only if the following character would be inter- preted as part of the number or name. The number may be zero to include - the entire matched string. For example, if the pattern a(b)c is - matched with "=abc=" and the replacement string "+$1$0$1+", the result + the entire matched string. For example, if the pattern a(b)c is + matched with "=abc=" and the replacement string "+$1$0$1+", the result is "=+babcb+=". - $*MARK inserts the name from the last encountered backtracking control - verb on the matching path that has a name. (*MARK) must always include - a name, but the other verbs need not. For example, in the case of + $*MARK inserts the name from the last encountered backtracking control + verb on the matching path that has a name. (*MARK) must always include + a name, but the other verbs need not. For example, in the case of (*MARK:A)(*PRUNE) the name inserted is "A", but for (*MARK:A)(*PRUNE:B) - the relevant name is "B". This facility can be used to perform simple + the relevant name is "B". This facility can be used to perform simple simultaneous substitutions, as this pcre2test example shows: /(*MARK:pear)apple|(*MARK:orange)lemon/g,replace=${*MARK} apple lemon 2: pear orange - As well as the usual options for pcre2_match(), a number of additional - options can be set in the options argument of pcre2_substitute(). - PCRE2_SUBSTITUTE_GLOBAL causes the function to iterate over the subject string, replacing every matching substring. If this option is not set, only the first matching substring is replaced. The search for matches @@ -3253,33 +3406,17 @@ CREATING A NEW STRING WITH SUBSTITUTIONS two characters are CR, LF. In this case, the offset is advanced by two characters. - PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output - buffer is too small. The default action is to return PCRE2_ERROR_NOMEM- - ORY immediately. If this option is set, however, pcre2_substitute() - continues to go through the motions of matching and substituting (with- - out, of course, writing anything) in order to compute the size of buf- - fer that is needed. This value is passed back via the outlengthptr - variable, with the result of the function still being - PCRE2_ERROR_NOMEMORY. - - Passing a buffer size of zero is a permitted way of finding out how - much memory is needed for given substitution. However, this does mean - that the entire operation is carried out twice. Depending on the appli- - cation, it may be more efficient to allocate a large buffer and free - the excess afterwards, instead of using PCRE2_SUBSTITUTE_OVER- - FLOW_LENGTH. - PCRE2_SUBSTITUTE_UNKNOWN_UNSET causes references to capture groups that do not appear in the pattern to be treated as unset groups. This option should be used with care, because it means that a typo in a group name or number no longer causes the PCRE2_ERROR_NOSUBSTRING error. - PCRE2_SUBSTITUTE_UNSET_EMPTY causes unset capture groups (including - unknown groups when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) to be - treated as empty strings when inserted as described above. If this - option is not set, an attempt to insert an unset group causes the - PCRE2_ERROR_UNSET error. This option does not influence the extended - substitution syntax described below. + PCRE2_SUBSTITUTE_UNSET_EMPTY causes unset capture groups (including un- + known groups when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) to be treated + as empty strings when inserted as described above. If this option is + not set, an attempt to insert an unset group causes the PCRE2_ERROR_UN- + SET error. This option does not influence the extended substitution + syntax described below. PCRE2_SUBSTITUTE_EXTENDED causes extra processing to be applied to the replacement string. Without this option, only the dollar character is @@ -3301,33 +3438,34 @@ CREATING A NEW STRING WITH SUBSTITUTIONS it is a letter) to upper or lower case, respectively, and then the state automatically reverts to no case forcing. Case forcing applies to all inserted characters, including those from capture groups and let- - ters within \Q...\E quoted sequences. + ters within \Q...\E quoted sequences. If either PCRE2_UTF or PCRE2_UCP + was set when the pattern was compiled, Unicode properties are used for + case forcing characters whose code points are greater than 127. Note that case forcing sequences such as \U...\E do not nest. For exam- ple, the result of processing "\Uaa\LBB\Ecc\E" is "AAbbcc"; the final - \E has no effect. Note also that the PCRE2_ALT_BSUX and - PCRE2_EXTRA_ALT_BSUX options do not apply to not apply to replacement - strings. + \E has no effect. Note also that the PCRE2_ALT_BSUX and PCRE2_EX- + TRA_ALT_BSUX options do not apply to replacement strings. - The second effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more - flexibility to capture group substitution. The syntax is similar to + The second effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more + flexibility to capture group substitution. The syntax is similar to that used by Bash: ${<n>:-<string>} ${<n>:+<string1>:<string2>} - As before, <n> may be a group number or a name. The first form speci- - fies a default value. If group <n> is set, its value is inserted; if - not, <string> is expanded and the result inserted. The second form - specifies strings that are expanded and inserted when group <n> is set - or unset, respectively. The first form is just a convenient shorthand + As before, <n> may be a group number or a name. The first form speci- + fies a default value. If group <n> is set, its value is inserted; if + not, <string> is expanded and the result inserted. The second form + specifies strings that are expanded and inserted when group <n> is set + or unset, respectively. The first form is just a convenient shorthand for ${<n>:+${<n>}:<string>} - Backslash can be used to escape colons and closing curly brackets in - the replacement strings. A change of the case forcing state within a - replacement string remains in force afterwards, as shown in this + Backslash can be used to escape colons and closing curly brackets in + the replacement strings. A change of the case forcing state within a + replacement string remains in force afterwards, as shown in this pcre2test example: /(some)?(body)/substitute_extended,replace=${1:+\U:\L}HeLLo @@ -3336,43 +3474,48 @@ CREATING A NEW STRING WITH SUBSTITUTIONS somebody 1: HELLO - The PCRE2_SUBSTITUTE_UNSET_EMPTY option does not affect these extended - substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause - unknown groups in the extended syntax forms to be treated as unset. + The PCRE2_SUBSTITUTE_UNSET_EMPTY option does not affect these extended + substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause un- + known groups in the extended syntax forms to be treated as unset. + + If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_UNKNOWN_UNSET, + PCRE2_SUBSTITUTE_UNSET_EMPTY, and PCRE2_SUBSTITUTE_EXTENDED are irrele- + vant and are ignored. - If successful, pcre2_substitute() returns the number of successful - matches. This may be zero if no matches were found, and is never - greater than 1 unless PCRE2_SUBSTITUTE_GLOBAL is set. + Substitution errors - In the event of an error, a negative error code is returned. Except for - PCRE2_ERROR_NOMATCH (which is never returned), errors from - pcre2_match() are passed straight back. + In the event of an error, pcre2_substitute() returns a negative error + code. Except for PCRE2_ERROR_NOMATCH (which is never returned), errors + from pcre2_match() are passed straight back. PCRE2_ERROR_NOSUBSTRING is returned for a non-existent substring inser- tion, unless PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set. PCRE2_ERROR_UNSET is returned for an unset substring insertion (includ- - ing an unknown substring when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) - when the simple (non-extended) syntax is used and PCRE2_SUBSTI- - TUTE_UNSET_EMPTY is not set. + ing an unknown substring when PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set) + when the simple (non-extended) syntax is used and PCRE2_SUBSTITUTE_UN- + SET_EMPTY is not set. - PCRE2_ERROR_NOMEMORY is returned if the output buffer is not big + PCRE2_ERROR_NOMEMORY is returned if the output buffer is not big enough. If the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set, the size - of buffer that is needed is returned via outlengthptr. Note that this + of buffer that is needed is returned via outlengthptr. Note that this does not happen by default. + PCRE2_ERROR_NULL is returned if PCRE2_SUBSTITUTE_MATCHED is set but the + match_data argument is NULL. + PCRE2_ERROR_BADREPLACEMENT is used for miscellaneous syntax errors in - the replacement string, with more particular errors being - PCRE2_ERROR_BADREPESCAPE (invalid escape sequence), PCRE2_ERROR_REP- - MISSINGBRACE (closing curly bracket not found), PCRE2_ERROR_BADSUBSTI- - TUTION (syntax error in extended group substitution), and - PCRE2_ERROR_BADSUBSPATTERN (the pattern match ended before it started - or the match started earlier than the current position in the subject, - which can happen if \K is used in an assertion). + the replacement string, with more particular errors being PCRE2_ER- + ROR_BADREPESCAPE (invalid escape sequence), PCRE2_ERROR_REPMISSINGBRACE + (closing curly bracket not found), PCRE2_ERROR_BADSUBSTITUTION (syntax + error in extended group substitution), and PCRE2_ERROR_BADSUBSPATTERN + (the pattern match ended before it started or the match started earlier + than the current position in the subject, which can happen if \K is + used in an assertion). As for all PCRE2 errors, a text message that describes the error can be - obtained by calling the pcre2_get_error_message() function (see - "Obtaining a textual error message" above). + obtained by calling the pcre2_get_error_message() function (see "Ob- + taining a textual error message" above). Substitution callouts @@ -3422,8 +3565,8 @@ CREATING A NEW STRING WITH SUBSTITUTIONS If the value is zero, the replacement is accepted, and, if PCRE2_SUB- STITUTE_GLOBAL is set, processing continues with a search for the next - match. If the value is not zero, the current replacement is not - accepted. If the value is greater than zero, processing continues when + match. If the value is not zero, the current replacement is not ac- + cepted. If the value is greater than zero, processing continues when PCRE2_SUBSTITUTE_GLOBAL is set. Otherwise (the value is less than zero or PCRE2_SUBSTITUTE_GLOBAL is not set), the the rest of the input is copied to the output and the call to pcre2_substitute() exits, return- @@ -3436,10 +3579,10 @@ DUPLICATE CAPTURE GROUP NAMES PCRE2_SPTR name, PCRE2_SPTR *first, PCRE2_SPTR *last); When a pattern is compiled with the PCRE2_DUPNAMES option, names for - capture groups are not required to be unique. Duplicate names are - always allowed for groups with the same number, created by using the - (?| feature. Indeed, if such groups are named, they are required to use - the same names. + capture groups are not required to be unique. Duplicate names are al- + ways allowed for groups with the same number, created by using the (?| + feature. Indeed, if such groups are named, they are required to use the + same names. Normally, patterns that use duplicate names are such that in any one match, only one of each set of identically-named groups participates. @@ -3447,10 +3590,10 @@ DUPLICATE CAPTURE GROUP NAMES When duplicates are present, pcre2_substring_copy_byname() and pcre2_substring_get_byname() return the first substring corresponding - to the given name that is set. Only if none are set is - PCRE2_ERROR_UNSET is returned. The pcre2_substring_number_from_name() - function returns the error PCRE2_ERROR_NOUNIQUESUBSTRING when there are - duplicate names. + to the given name that is set. Only if none are set is PCRE2_ERROR_UN- + SET is returned. The pcre2_substring_number_from_name() function re- + turns the error PCRE2_ERROR_NOUNIQUESUBSTRING when there are duplicate + names. If you want to get full details of all captured substrings for a given name, you must use the pcre2_substring_nametable_scan() function. The @@ -3537,44 +3680,43 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION The unused bits of the options argument for pcre2_dfa_match() must be zero. The only bits that may be set are PCRE2_ANCHORED, - PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, - PCRE2_NOTEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, - PCRE2_NO_UTF_CHECK, PCRE2_PARTIAL_HARD, PCRE2_PARTIAL_SOFT, - PCRE2_DFA_SHORTEST, and PCRE2_DFA_RESTART. All but the last four of - these are exactly the same as for pcre2_match(), so their description - is not repeated here. + PCRE2_COPY_MATCHED_SUBJECT, PCRE2_ENDANCHORED, PCRE2_NOTBOL, PCRE2_NO- + TEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, PCRE2_NO_UTF_CHECK, + PCRE2_PARTIAL_HARD, PCRE2_PARTIAL_SOFT, PCRE2_DFA_SHORTEST, and + PCRE2_DFA_RESTART. All but the last four of these are exactly the same + as for pcre2_match(), so their description is not repeated here. PCRE2_PARTIAL_HARD PCRE2_PARTIAL_SOFT - These have the same general effect as they do for pcre2_match(), but - the details are slightly different. When PCRE2_PARTIAL_HARD is set for - pcre2_dfa_match(), it returns PCRE2_ERROR_PARTIAL if the end of the + These have the same general effect as they do for pcre2_match(), but + the details are slightly different. When PCRE2_PARTIAL_HARD is set for + pcre2_dfa_match(), it returns PCRE2_ERROR_PARTIAL if the end of the subject is reached and there is still at least one matching possibility that requires additional characters. This happens even if some complete - matches have already been found. When PCRE2_PARTIAL_SOFT is set, the - return code PCRE2_ERROR_NOMATCH is converted into PCRE2_ERROR_PARTIAL - if the end of the subject is reached, there have been no complete + matches have already been found. When PCRE2_PARTIAL_SOFT is set, the + return code PCRE2_ERROR_NOMATCH is converted into PCRE2_ERROR_PARTIAL + if the end of the subject is reached, there have been no complete matches, but there is still at least one matching possibility. The por- - tion of the string that was inspected when the longest partial match + tion of the string that was inspected when the longest partial match was found is set as the first matching string in both cases. There is a - more detailed discussion of partial and multi-segment matching, with + more detailed discussion of partial and multi-segment matching, with examples, in the pcre2partial documentation. PCRE2_DFA_SHORTEST - Setting the PCRE2_DFA_SHORTEST option causes the matching algorithm to + Setting the PCRE2_DFA_SHORTEST option causes the matching algorithm to stop as soon as it has found one match. Because of the way the alterna- - tive algorithm works, this is necessarily the shortest possible match + tive algorithm works, this is necessarily the shortest possible match at the first possible matching point in the subject string. PCRE2_DFA_RESTART - When pcre2_dfa_match() returns a partial match, it is possible to call + When pcre2_dfa_match() returns a partial match, it is possible to call it again, with additional subject characters, and have it continue with the same match. The PCRE2_DFA_RESTART option requests this action; when - it is set, the workspace and wscount options must reference the same - vector as before because data about the match so far is left in them + it is set, the workspace and wscount options must reference the same + vector as before because data about the match so far is left in them after a partial match. There is more discussion of this facility in the pcre2partial documentation. @@ -3582,8 +3724,8 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION When pcre2_dfa_match() succeeds, it may have matched more than one sub- string in the subject. Note, however, that all the matches from one run - of the function start at the same point in the subject. The shorter - matches are all initial substrings of the longer matches. For example, + of the function start at the same point in the subject. The shorter + matches are all initial substrings of the longer matches. For example, if the pattern <.*> @@ -3598,74 +3740,80 @@ MATCHING A PATTERN: THE ALTERNATIVE FUNCTION <something> <something else> <something> - On success, the yield of the function is a number greater than zero, - which is the number of matched substrings. The offsets of the sub- - strings are returned in the ovector, and can be extracted by number in - the same way as for pcre2_match(), but the numbers bear no relation to - any capture groups that may exist in the pattern, because DFA matching + On success, the yield of the function is a number greater than zero, + which is the number of matched substrings. The offsets of the sub- + strings are returned in the ovector, and can be extracted by number in + the same way as for pcre2_match(), but the numbers bear no relation to + any capture groups that may exist in the pattern, because DFA matching does not support capturing. - Calls to the convenience functions that extract substrings by name - return the error PCRE2_ERROR_DFA_UFUNC (unsupported function) if used - after a DFA match. The convenience functions that extract substrings by + Calls to the convenience functions that extract substrings by name re- + turn the error PCRE2_ERROR_DFA_UFUNC (unsupported function) if used af- + ter a DFA match. The convenience functions that extract substrings by number never return PCRE2_ERROR_NOSUBSTRING. - The matched strings are stored in the ovector in reverse order of - length; that is, the longest matching string is first. If there were - too many matches to fit into the ovector, the yield of the function is + The matched strings are stored in the ovector in reverse order of + length; that is, the longest matching string is first. If there were + too many matches to fit into the ovector, the yield of the function is zero, and the vector is filled with the longest matches. - NOTE: PCRE2's "auto-possessification" optimization usually applies to - character repeats at the end of a pattern (as well as internally). For - example, the pattern "a\d+" is compiled as if it were "a\d++". For DFA - matching, this means that only one possible match is found. If you - really do want multiple matches in such cases, either use an ungreedy - repeat such as "a\d+?" or set the PCRE2_NO_AUTO_POSSESS option when - compiling. + NOTE: PCRE2's "auto-possessification" optimization usually applies to + character repeats at the end of a pattern (as well as internally). For + example, the pattern "a\d+" is compiled as if it were "a\d++". For DFA + matching, this means that only one possible match is found. If you re- + ally do want multiple matches in such cases, either use an ungreedy re- + peat such as "a\d+?" or set the PCRE2_NO_AUTO_POSSESS option when com- + piling. Error returns from pcre2_dfa_match() The pcre2_dfa_match() function returns a negative number when it fails. - Many of the errors are the same as for pcre2_match(), as described + Many of the errors are the same as for pcre2_match(), as described above. There are in addition the following errors that are specific to pcre2_dfa_match(): PCRE2_ERROR_DFA_UITEM - This return is given if pcre2_dfa_match() encounters an item in the - pattern that it does not support, for instance, the use of \C in a UTF + This return is given if pcre2_dfa_match() encounters an item in the + pattern that it does not support, for instance, the use of \C in a UTF mode or a backreference. PCRE2_ERROR_DFA_UCOND - This return is given if pcre2_dfa_match() encounters a condition item + This return is given if pcre2_dfa_match() encounters a condition item that uses a backreference for the condition, or a test for recursion in a specific capture group. These are not supported. + PCRE2_ERROR_DFA_UINVALID_UTF + + This return is given if pcre2_dfa_match() is called for a pattern that + was compiled with PCRE2_MATCH_INVALID_UTF. This is not supported for + DFA matching. + PCRE2_ERROR_DFA_WSSIZE - This return is given if pcre2_dfa_match() runs out of space in the + This return is given if pcre2_dfa_match() runs out of space in the workspace vector. PCRE2_ERROR_DFA_RECURSE When a recursion or subroutine call is processed, the matching function - calls itself recursively, using private memory for the ovector and - workspace. This error is given if the internal ovector is not large - enough. This should be extremely rare, as a vector of size 1000 is + calls itself recursively, using private memory for the ovector and + workspace. This error is given if the internal ovector is not large + enough. This should be extremely rare, as a vector of size 1000 is used. PCRE2_ERROR_DFA_BADRESTART - When pcre2_dfa_match() is called with the PCRE2_DFA_RESTART option, - some plausibility checks are made on the contents of the workspace, - which should contain data about the previous partial match. If any of + When pcre2_dfa_match() is called with the PCRE2_DFA_RESTART option, + some plausibility checks are made on the contents of the workspace, + which should contain data about the previous partial match. If any of these checks fail, this error is given. SEE ALSO - pcre2build(3), pcre2callout(3), pcre2demo(3), pcre2matching(3), + pcre2build(3), pcre2callout(3), pcre2demo(3), pcre2matching(3), pcre2partial(3), pcre2posix(3), pcre2sample(3), pcre2unicode(3). @@ -3678,8 +3826,8 @@ AUTHOR REVISION - Last updated: 14 February 2019 - Copyright (c) 1997-2019 University of Cambridge. + Last updated: 04 November 2020 + Copyright (c) 1997-2020 University of Cambridge. ------------------------------------------------------------------------------ @@ -3695,9 +3843,9 @@ BUILDING PCRE2 PCRE2 is distributed with a configure script that can be used to build the library in Unix-like environments using the applications known as Autotools. Also in the distribution are files to support building using - CMake instead of configure. The text file README contains general - information about building with Autotools (some of which is repeated - below), and also has some comments about building on various operating + CMake instead of configure. The text file README contains general in- + formation about building with Autotools (some of which is repeated be- + low), and also has some comments about building on various operating systems. There is a lot more information about building PCRE2 without using Autotools (including information about using CMake and building "by hand") in the text file called NON-AUTOTOOLS-BUILD. You should @@ -3720,8 +3868,8 @@ PCRE2 BUILD-TIME OPTIONS compiler, as described in NON-AUTOTOOLS-BUILD. The complete list of options for configure (which includes the standard - ones such as the selection of the installation directory) can be - obtained by running + ones such as the selection of the installation directory) can be ob- + tained by running ./configure --help @@ -3778,8 +3926,8 @@ UNICODE AND UTF SUPPORT --disable-unicode to the configure command. This setting applies to all three libraries. - It is not possible to build one library with Unicode support, and - another without, in the same configuration. + It is not possible to build one library with Unicode support and an- + other without in the same configuration. Of itself, Unicode support does not make PCRE2 treat strings as UTF-8, UTF-16 or UTF-32. To do that, applications that use the library can set @@ -3805,8 +3953,8 @@ DISABLING THE USE OF \C The \C escape sequence, which matches a single code unit, even in a UTF mode, can cause unpredictable behaviour because it may leave the cur- rent matching point in the middle of a multi-code-unit character. The - application can lock it out by setting the PCRE2_NEVER_BACKSLASH_C - option when calling pcre2_compile(). There is also a build-time option + application can lock it out by setting the PCRE2_NEVER_BACKSLASH_C op- + tion when calling pcre2_compile(). There is also a build-time option --enable-never-backslash-C @@ -3840,7 +3988,7 @@ JUST-IN-TIME COMPILER SUPPORT --disable-pcre2grep-jit - to the "configure" command. + to the configure command. NEWLINE RECOGNITION @@ -3852,8 +4000,8 @@ NEWLINE RECOGNITION --enable-newline-is-cr - to the configure command. There is also an --enable-newline-is-lf - option, which explicitly specifies linefeed as the newline character. + to the configure command. There is also an --enable-newline-is-lf op- + tion, which explicitly specifies linefeed as the newline character. Alternatively, you can specify that line endings are to be indicated by the two-character sequence CRLF (CR immediately followed by LF). If you @@ -3965,8 +4113,8 @@ LIMITING PCRE2 RESOURCE USAGE used, but because the size of each backtracking "frame" depends on the number of capturing parentheses in a pattern, the amount of heap that is used before the limit is reached varies from pattern to pattern. - This limit was more useful in versions before 10.30, where function - recursion was used for backtracking. + This limit was more useful in versions before 10.30, where function re- + cursion was used for backtracking. As well as applying to pcre2_match(), the depth limit also controls the depth of recursive function calls in pcre2_dfa_match(). These are used @@ -3984,39 +4132,61 @@ CREATING CHARACTER TABLES AT BUILD TIME --enable-rebuild-chartables to the configure command, the distributed tables are no longer used. - Instead, a program called dftables is compiled and run. This outputs - the source for new set of tables, created in the default locale of your - C run-time system. This method of replacing the tables does not work if - you are cross compiling, because dftables is run on the local host. If - you need to create alternative tables when cross compiling, you will - have to do so "by hand". + Instead, a program called pcre2_dftables is compiled and run. This out- + puts the source for new set of tables, created in the default locale of + your C run-time system. This method of replacing the tables does not + work if you are cross compiling, because pcre2_dftables needs to be run + on the local host and therefore not compiled with the cross compiler. + + If you need to create alternative tables when cross compiling, you will + have to do so "by hand". There may also be other reasons for creating + tables manually. To cause pcre2_dftables to be built on the local + host, run a normal compiling command, and then run the program with the + output file as its argument, for example: + + cc src/pcre2_dftables.c -o pcre2_dftables + ./pcre2_dftables src/pcre2_chartables.c + + This builds the tables in the default locale of the local host. If you + want to specify a locale, you must use the -L option: + + LC_ALL=fr_FR ./pcre2_dftables -L src/pcre2_chartables.c + + You can also specify -b (with or without -L). This causes the tables to + be written in binary instead of as source code. A set of binary tables + can be loaded into memory by an application and passed to pcre2_com- + pile() in the same way as tables created by calling pcre2_maketables(). + The tables are just a string of bytes, independent of hardware charac- + teristics such as endianness. This means they can be bundled with an + application that runs in different environments, to ensure consistent + behaviour. USING EBCDIC CODE - PCRE2 assumes by default that it will run in an environment where the - character code is ASCII or Unicode, which is a superset of ASCII. This + PCRE2 assumes by default that it will run in an environment where the + character code is ASCII or Unicode, which is a superset of ASCII. This is the case for most computer operating systems. PCRE2 can, however, be compiled to run in an 8-bit EBCDIC environment by adding --enable-ebcdic --disable-unicode to the configure command. This setting implies --enable-rebuild-charta- - bles. You should only use it if you know that you are in an EBCDIC - environment (for example, an IBM mainframe operating system). + bles. You should only use it if you know that you are in an EBCDIC en- + vironment (for example, an IBM mainframe operating system). - It is not possible to support both EBCDIC and UTF-8 codes in the same - version of the library. Consequently, --enable-unicode and --enable- + It is not possible to support both EBCDIC and UTF-8 codes in the same + version of the library. Consequently, --enable-unicode and --enable- ebcdic are mutually exclusive. The EBCDIC character that corresponds to an ASCII LF is assumed to have - the value 0x15 by default. However, in some EBCDIC environments, 0x25 + the value 0x15 by default. However, in some EBCDIC environments, 0x25 is used. In such an environment you should use --enable-ebcdic-nl25 as well as, or instead of, --enable-ebcdic. The EBCDIC character for CR - has the same value as in ASCII, namely, 0x0d. Whichever of 0x15 and + has the same value as in ASCII, namely, 0x0d. Whichever of 0x15 and 0x25 is not chosen as LF is made to correspond to the Unicode NEL char- acter (which, in Unicode, is 0x85). @@ -4028,47 +4198,47 @@ USING EBCDIC CODE PCRE2GREP SUPPORT FOR EXTERNAL SCRIPTS By default pcre2grep supports the use of callouts with string arguments - within the patterns it is matching. There are two kinds: one that gen- + within the patterns it is matching. There are two kinds: one that gen- erates output using local code, and another that calls an external pro- - gram or script. If --disable-pcre2grep-callout-fork is added to the - configure command, only the first kind of callout is supported; if - --disable-pcre2grep-callout is used, all callouts are completely - ignored. For more details of pcre2grep callouts, see the pcre2grep doc- - umentation. + gram or script. If --disable-pcre2grep-callout-fork is added to the + configure command, only the first kind of callout is supported; if + --disable-pcre2grep-callout is used, all callouts are completely ig- + nored. For more details of pcre2grep callouts, see the pcre2grep docu- + mentation. PCRE2GREP OPTIONS FOR COMPRESSED FILE SUPPORT - By default, pcre2grep reads all files as plain text. You can build it - so that it recognizes files whose names end in .gz or .bz2, and reads + By default, pcre2grep reads all files as plain text. You can build it + so that it recognizes files whose names end in .gz or .bz2, and reads them with libz or libbz2, respectively, by adding one or both of --enable-pcre2grep-libz --enable-pcre2grep-libbz2 to the configure command. These options naturally require that the rel- - evant libraries are installed on your system. Configuration will fail + evant libraries are installed on your system. Configuration will fail if they are not. PCRE2GREP BUFFER SIZE - pcre2grep uses an internal buffer to hold a "window" on the file it is + pcre2grep uses an internal buffer to hold a "window" on the file it is scanning, in order to be able to output "before" and "after" lines when it finds a match. The default starting size of the buffer is 20KiB. The - buffer itself is three times this size, but because of the way it is + buffer itself is three times this size, but because of the way it is used for holding "before" lines, the longest line that is guaranteed to be processable is the notional buffer size. If a longer line is encoun- - tered, pcre2grep automatically expands the buffer, up to a specified - maximum size, whose default is 1MiB or the starting size, whichever is - the larger. You can change the default parameter values by adding, for + tered, pcre2grep automatically expands the buffer, up to a specified + maximum size, whose default is 1MiB or the starting size, whichever is + the larger. You can change the default parameter values by adding, for example, --with-pcre2grep-bufsize=51200 --with-pcre2grep-max-bufsize=2097152 - to the configure command. The caller of pcre2grep can override these - values by using --buffer-size and --max-buffer-size on the command + to the configure command. The caller of pcre2grep can override these + values by using --buffer-size and --max-buffer-size on the command line. @@ -4079,26 +4249,26 @@ PCRE2TEST OPTION FOR LIBREADLINE SUPPORT --enable-pcre2test-libreadline --enable-pcre2test-libedit - to the configure command, pcre2test is linked with the libreadline - orlibedit library, respectively, and when its input is from a terminal, - it reads it using the readline() function. This provides line-editing - and history facilities. Note that libreadline is GPL-licensed, so if - you distribute a binary of pcre2test linked in this way, there may be + to the configure command, pcre2test is linked with the libreadline or- + libedit library, respectively, and when its input is from a terminal, + it reads it using the readline() function. This provides line-editing + and history facilities. Note that libreadline is GPL-licensed, so if + you distribute a binary of pcre2test linked in this way, there may be licensing issues. These can be avoided by linking instead with libedit, which has a BSD licence. - Setting --enable-pcre2test-libreadline causes the -lreadline option to - be added to the pcre2test build. In many operating environments with a - sytem-installed readline library this is sufficient. However, in some + Setting --enable-pcre2test-libreadline causes the -lreadline option to + be added to the pcre2test build. In many operating environments with a + sytem-installed readline library this is sufficient. However, in some environments (e.g. if an unmodified distribution version of readline is - in use), some extra configuration may be necessary. The INSTALL file + in use), some extra configuration may be necessary. The INSTALL file for libreadline says this: "Readline uses the termcap functions, but does not link with the termcap or curses library itself, allowing applications which link with readline the to choose an appropriate library." - If your environment has not been set up so that an appropriate library + If your environment has not been set up so that an appropriate library is automatically included, you may need to add something like LIBS="-ncurses" @@ -4112,7 +4282,7 @@ INCLUDING DEBUGGING CODE --enable-debug - to the configure command, additional debugging code is included in the + to the configure command, additional debugging code is included in the build. This feature is intended for use by the PCRE2 maintainers. @@ -4122,10 +4292,9 @@ DEBUGGING WITH VALGRIND SUPPORT --enable-valgrind - to the configure command, PCRE2 will use valgrind annotations to mark - certain memory regions as unaddressable. This allows it to detect - invalid memory accesses, and is mostly useful for debugging PCRE2 - itself. + to the configure command, PCRE2 will use valgrind annotations to mark + certain memory regions as unaddressable. This allows it to detect in- + valid memory accesses, and is mostly useful for debugging PCRE2 itself. CODE COVERAGE REPORTING @@ -4189,13 +4358,13 @@ DISABLING THE Z AND T FORMATTING MODIFIERS The C99 standard defines formatting modifiers z and t for size_t and ptrdiff_t values, respectively. By default, PCRE2 uses these modifiers in environments other than Microsoft Visual Studio when __STDC_VER- - SION__ is defined and has a value greater than or equal to 199901L - (indicating C99). However, there is at least one environment that - claims to be C99 but does not support these modifiers. If + SION__ is defined and has a value greater than or equal to 199901L (in- + dicating C99). However, there is at least one environment that claims + to be C99 but does not support these modifiers. If --disable-percent-zt - is specified, no use is made of the z or t modifiers. Instead or %td or + is specified, no use is made of the z or t modifiers. Instead of %td or %zu, %lu is used, with a cast for size_t values. @@ -4207,8 +4376,8 @@ SUPPORT FOR FUZZERS --enable-fuzz-support At present this applies only to the 8-bit library. If set, it causes an - extra library called libpcre2-fuzzsupport.a to be built, but not - installed. This contains a single function called LLVMFuzzerTestOneIn- + extra library called libpcre2-fuzzsupport.a to be built, but not in- + stalled. This contains a single function called LLVMFuzzerTestOneIn- put() whose arguments are a pointer to a string and the length of the string. When called, this function tries to compile the string as a pattern, and if that succeeds, to match it. This is done both with no @@ -4251,8 +4420,8 @@ AUTHOR REVISION - Last updated: 03 March 2019 - Copyright (c) 1997-2019 University of Cambridge. + Last updated: 20 March 2020 + Copyright (c) 1997-2020 University of Cambridge. ------------------------------------------------------------------------------ @@ -4400,8 +4569,8 @@ MISSING CALLOUTS No match This shows that all match attempts start at the beginning of the sub- - ject. In other words, the pattern is anchored. You can disable this - optimization by passing PCRE2_NO_DOTSTAR_ANCHOR to pcre2_compile(), or + ject. In other words, the pattern is anchored. You can disable this op- + timization by passing PCRE2_NO_DOTSTAR_ANCHOR to pcre2_compile(), or starting the pattern with (*NO_DOTSTAR_ANCHOR). In this case, the out- put changes to: @@ -4512,10 +4681,10 @@ THE CALLOUT INTERFACE (the "ovector"). You may read the elements in this vector, but you must not change any of them. - For calls to pcre2_match(), the offset_vector field is not (since - release 10.30) a pointer to the actual ovector that was passed to the - matching function in the match data block. Instead it points to an - internal ovector of a size large enough to hold all possible captured + For calls to pcre2_match(), the offset_vector field is not (since re- + lease 10.30) a pointer to the actual ovector that was passed to the + matching function in the match data block. Instead it points to an in- + ternal ovector of a size large enough to hold all possible captured substrings in the pattern. Note that whenever a recursion or subroutine call within a pattern completes, the capturing state is reset to what it was before. @@ -4528,8 +4697,8 @@ THE CALLOUT INTERFACE differ by one; for example, when the callout in the pattern ((a)(b))(?C2) is taken, capture_last is 1 but capture_top is 4. - The contents of ovector[2] to ovector[<capture_top>*2-1] can be - inspected in order to extract substrings that have been matched so far, + The contents of ovector[2] to ovector[<capture_top>*2-1] can be in- + spected in order to extract substrings that have been matched so far, in the same way as extracting substrings after a match has completed. The values in ovector[0] and ovector[1] are always PCRE2_UNSET because the match is by definition not complete. Substrings that have not been @@ -4548,8 +4717,8 @@ THE CALLOUT INTERFACE were passed to the matching function. The start_match field normally contains the offset within the subject - at which the current match attempt started. However, if the escape - sequence \K has been encountered, this value is changed to reflect the + at which the current match attempt started. However, if the escape se- + quence \K has been encountered, this value is changed to reflect the modified starting point. If the pattern is not anchored, the callout function may be called several times from the same point in the pattern for different starting points in the subject. @@ -4623,11 +4792,10 @@ RETURN VALUES FROM CALLOUTS failed. If the value is less than zero, the match is abandoned, and the matching function returns the negative value. - Negative values should normally be chosen from the set of - PCRE2_ERROR_xxx values. In particular, PCRE2_ERROR_NOMATCH forces a - standard "no match" failure. The error number PCRE2_ERROR_CALLOUT is - reserved for use by callout functions; it will never be used by PCRE2 - itself. + Negative values should normally be chosen from the set of PCRE2_ER- + ROR_xxx values. In particular, PCRE2_ERROR_NOMATCH forces a standard + "no match" failure. The error number PCRE2_ERROR_CALLOUT is reserved + for use by callout functions; it will never be used by PCRE2 itself. CALLOUT ENUMERATION @@ -4637,14 +4805,14 @@ CALLOUT ENUMERATION void *user_data); A script language that supports the use of string arguments in callouts - might like to scan all the callouts in a pattern before running the + might like to scan all the callouts in a pattern before running the match. This can be done by calling pcre2_callout_enumerate(). The first - argument is a pointer to a compiled pattern, the second points to a - callback function, and the third is arbitrary user data. The callback - function is called for every callout in the pattern in the order in + argument is a pointer to a compiled pattern, the second points to a + callback function, and the third is arbitrary user data. The callback + function is called for every callout in the pattern in the order in which they appear. Its first argument is a pointer to a callout enumer- - ation block, and its second argument is the user_data value that was - passed to pcre2_callout_enumerate(). The data block contains the fol- + ation block, and its second argument is the user_data value that was + passed to pcre2_callout_enumerate(). The data block contains the fol- lowing fields: version Block version number @@ -4655,17 +4823,17 @@ CALLOUT ENUMERATION callout_string_length Length of callout string callout_string Points to callout string or is NULL - The version number is currently 0. It will increase if new fields are - ever added to the block. The remaining fields are the same as their - namesakes in the pcre2_callout block that is used for callouts during + The version number is currently 0. It will increase if new fields are + ever added to the block. The remaining fields are the same as their + namesakes in the pcre2_callout block that is used for callouts during matching, as described above. - Note that the value of pattern_position is unique for each callout. - However, if a callout occurs inside a group that is quantified with a + Note that the value of pattern_position is unique for each callout. + However, if a callout occurs inside a group that is quantified with a non-zero minimum or a fixed maximum, the group is replicated inside the - compiled pattern. For example, a pattern such as /(a){2}/ is compiled - as if it were /(a)(a)/. This means that the callout will be enumerated - more than once, but with the same value for pattern_position in each + compiled pattern. For example, a pattern such as /(a){2}/ is compiled + as if it were /(a)(a)/. This means that the callout will be enumerated + more than once, but with the same value for pattern_position in each case. The callback function should normally return zero. If it returns a non- @@ -4696,56 +4864,59 @@ NAME DIFFERENCES BETWEEN PCRE2 AND PERL - This document describes the differences in the ways that PCRE2 and Perl - handle regular expressions. The differences described here are with - respect to Perl versions 5.26, but as both Perl and PCRE2 are continu- - ally changing, the information may sometimes be out of date. + This document describes some of the differences in the ways that PCRE2 + and Perl handle regular expressions. The differences described here are + with respect to Perl version 5.32.0, but as both Perl and PCRE2 are + continually changing, the information may at times be out of date. 1. PCRE2 has only a subset of Perl's Unicode support. Details of what it does have are given in the pcre2unicode page. 2. Like Perl, PCRE2 allows repeat quantifiers on parenthesized asser- tions, but they do not mean what you might think. For example, (?!a){3} - does not assert that the next three characters are not "a". It just - asserts that the next character is not "a" three times (in principle; + does not assert that the next three characters are not "a". It just as- + serts that the next character is not "a" three times (in principle; PCRE2 optimizes this to run the assertion just once). Perl allows some repeat quantifiers on other assertions, for example, \b* (but not - \b{3}), but these do not seem to have any use. + \b{3}, though oddly it does allow ^{3}), but these do not seem to have + any use. PCRE2 does not allow any kind of quantifier on non-lookaround + assertions. 3. Capture groups that occur inside negative lookaround assertions are counted, but their entries in the offsets vector are set only when a negative assertion is a condition that has a matching branch (that is, - the condition is false). + the condition is false). Perl may set such capture groups in other + circumstances. - 4. The following Perl escape sequences are not supported: \F, \l, \L, + 4. The following Perl escape sequences are not supported: \F, \l, \L, \u, \U, and \N when followed by a character name. \N on its own, match- - ing a non-newline character, and \N{U+dd..}, matching a Unicode code - point, are supported. The escapes that modify the case of following - letters are implemented by Perl's general string-handling and are not + ing a non-newline character, and \N{U+dd..}, matching a Unicode code + point, are supported. The escapes that modify the case of following + letters are implemented by Perl's general string-handling and are not part of its pattern matching engine. If any of these are encountered by - PCRE2, an error is generated by default. However, if either of the - PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX options is set, \U and \u are + PCRE2, an error is generated by default. However, if either of the + PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX options is set, \U and \u are interpreted as ECMAScript interprets them. 5. The Perl escape sequences \p, \P, and \X are supported only if PCRE2 is built with Unicode support (the default). The properties that can be - tested with \p and \P are limited to the general category properties - such as Lu and Nd, script names such as Greek or Han, and the derived - properties Any and L&. PCRE2 does support the Cs (surrogate) property, - which Perl does not; the Perl documentation says "Because Perl hides - the need for the user to understand the internal representation of Uni- - code characters, there is no need to implement the somewhat messy con- - cept of surrogates." + tested with \p and \P are limited to the general category properties + such as Lu and Nd, script names such as Greek or Han, and the derived + properties Any and L&. Both PCRE2 and Perl support the Cs (surrogate) + property, but in PCRE2 its use is limited. See the pcre2pattern docu- + mentation for details. The long synonyms for property names that Perl + supports (such as \p{Letter}) are not supported by PCRE2, nor is it + permitted to prefix any of these properties with "Is". 6. PCRE2 supports the \Q...\E escape for quoting substrings. Characters in between are treated as literals. However, this is slightly different - from Perl in that $ and @ are also handled as literals inside the + from Perl in that $ and @ are also handled as literals inside the quotes. In Perl, they cause variable interpolation (but of course PCRE2 - does not have variables). Also, Perl does "double-quotish backslash - interpolation" on any backslashes between \Q and \E which, its documen- - tation says, "may lead to confusing results". PCRE2 treats a backslash - between \Q and \E just like any other character. Note the following - examples: + does not have variables). Also, Perl does "double-quotish backslash in- + terpolation" on any backslashes between \Q and \E which, its documenta- + tion says, "may lead to confusing results". PCRE2 treats a backslash + between \Q and \E just like any other character. Note the following ex- + amples: Pattern PCRE2 matches Perl matches @@ -4756,78 +4927,79 @@ DIFFERENCES BETWEEN PCRE2 AND PERL \QA\B\E A\B A\B \Q\\E \ \\E - The \Q...\E sequence is recognized both inside and outside character - classes. + The \Q...\E sequence is recognized both inside and outside character + classes by both PCRE2 and Perl. - 7. Fairly obviously, PCRE2 does not support the (?{code}) and + 7. Fairly obviously, PCRE2 does not support the (?{code}) and (??{code}) constructions. However, PCRE2 does have a "callout" feature, which allows an external function to be called during pattern matching. See the pcre2callout documentation for details. - 8. Subroutine calls (whether recursive or not) were treated as atomic - groups up to PCRE2 release 10.23, but from release 10.30 this changed, + 8. Subroutine calls (whether recursive or not) were treated as atomic + groups up to PCRE2 release 10.23, but from release 10.30 this changed, and backtracking into subroutine calls is now supported, as in Perl. - 9. If any of the backtracking control verbs are used in a group that is - called as a subroutine (whether or not recursively), their effect is - confined to that group; it does not extend to the surrounding pattern. - This is not always the case in Perl. In particular, if (*THEN) is - present in a group that is called as a subroutine, its action is lim- - ited to that group, even if the group does not contain any | charac- - ters. Note that such groups are processed as anchored at the point - where they are tested. - - 10. If a pattern contains more than one backtracking control verb, the - first one that is backtracked onto acts. For example, in the pattern - A(*COMMIT)B(*PRUNE)C a failure in B triggers (*COMMIT), but a failure + 9. In PCRE2, if any of the backtracking control verbs are used in a + group that is called as a subroutine (whether or not recursively), + their effect is confined to that group; it does not extend to the sur- + rounding pattern. This is not always the case in Perl. In particular, + if (*THEN) is present in a group that is called as a subroutine, its + action is limited to that group, even if the group does not contain any + | characters. Note that such groups are processed as anchored at the + point where they are tested. + + 10. If a pattern contains more than one backtracking control verb, the + first one that is backtracked onto acts. For example, in the pattern + A(*COMMIT)B(*PRUNE)C a failure in B triggers (*COMMIT), but a failure in C triggers (*PRUNE). Perl's behaviour is more complex; in many cases it is the same as PCRE2, but there are cases where it differs. - 11. Most backtracking verbs in assertions have their normal actions. - They are not confined to the assertion. - - 12. There are some differences that are concerned with the settings of + 11. There are some differences that are concerned with the settings of captured strings when part of a pattern is repeated. For example, - matching "aba" against the pattern /^(a(b)?)+$/ in Perl leaves $2 - unset, but in PCRE2 it is set to "b". + matching "aba" against the pattern /^(a(b)?)+$/ in Perl leaves $2 un- + set, but in PCRE2 it is set to "b". - 13. PCRE2's handling of duplicate capture group numbers and names is + 12. PCRE2's handling of duplicate capture group numbers and names is not as general as Perl's. This is a consequence of the fact the PCRE2 works internally just with numbers, using an external table to trans- late between numbers and names. In particular, a pattern such as - (?|(?<a>A)|(?<b>B), where the two capture groups have the same number + (?|(?<a>A)|(?<b>B)), where the two capture groups have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which group matched, because both names map to capture group number 1. To avoid this confusing situation, an error is given at compile time. - 14. Perl used to recognize comments in some places that PCRE2 does not, + 13. Perl used to recognize comments in some places that PCRE2 does not, for example, between the ( and ? at the start of a group. If the /x modifier is set, Perl allowed white space between ( and ? though the latest Perls give an error (for a while it was just deprecated). There may still be some cases where Perl behaves differently. - 15. Perl, when in warning mode, gives warnings for character classes + 14. Perl, when in warning mode, gives warnings for character classes such as [A-\d] or [a-[:digit:]]. It then treats the hyphens as liter- als. PCRE2 has no warning features, so it gives an error in these cases because they are almost certainly user mistakes. - 16. In PCRE2, the upper/lower case character properties Lu and Ll are + 15. In PCRE2, the upper/lower case character properties Lu and Ll are not affected when case-independent matching is specified. For example, \p{Lu} always matches an upper case letter. I think Perl has changed in - this respect; in the release at the time of writing (5.24), \p{Lu} and + this respect; in the release at the time of writing (5.32), \p{Lu} and \p{Ll} match all letters, regardless of case, when case independence is specified. - 17. PCRE2 provides some extensions to the Perl regular expression - facilities. Perl 5.10 includes new features that are not in earlier + 16. From release 5.32.0, Perl locks out the use of \K in lookaround as- + sertions. In PCRE2, \K is acted on when it occurs in positive asser- + tions, but is ignored in negative assertions. + + 17. PCRE2 provides some extensions to the Perl regular expression fa- + cilities. Perl 5.10 included new features that were not in earlier versions of Perl, some of which (such as named parentheses) were in - PCRE2 for some time before. This list is with respect to Perl 5.26: + PCRE2 for some time before. This list is with respect to Perl 5.32: (a) Although lookbehind assertions in PCRE2 must match fixed length - strings, each alternative branch of a lookbehind assertion can match a - different length of string. Perl requires them all to have the same - length. + strings, each alternative toplevel branch of a lookbehind assertion can + match a different length of string. Perl requires them all to have the + same length. (b) From PCRE2 10.23, backreferences to groups of fixed length are sup- ported in lookbehinds, provided that there is no possibility of refer- @@ -4862,12 +5034,16 @@ DIFFERENCES BETWEEN PCRE2 AND PERL different way and is not Perl-compatible. (l) PCRE2 recognizes some special sequences such as (*CR) or (*NO_JIT) - at the start of a pattern that set overall options that cannot be + at the start of a pattern. These set overall options that cannot be changed within the pattern. + (m) PCRE2 supports non-atomic positive lookaround assertions. This is + an extension to the lookaround facilities. The default, Perl-compatible + lookarounds are atomic. + 18. The Perl /a modifier restricts /d numbers to pure ascii, and the - /aa modifier restricts /i case-insensitive matching to pure ascii, - ignoring Unicode rules. This separation cannot be represented with + /aa modifier restricts /i case-insensitive matching to pure ascii, ig- + noring Unicode rules. This separation cannot be represented with PCRE2_UCP. 19. Perl has different limits than PCRE2. See the pcre2limit documenta- @@ -4887,7 +5063,7 @@ AUTHOR REVISION - Last updated: 12 February 2019 + Last updated: 06 October 2020 Copyright (c) 1997-2019 University of Cambridge. ------------------------------------------------------------------------------ @@ -4960,8 +5136,8 @@ SIMPLE USE OF JIT There is a limit to the size of pattern that JIT supports, imposed by the size of machine stack that it uses. The exact rules are not docu- - mented because they may change at any time, in particular, when new - optimizations are introduced. If a pattern is too big, a call to + mented because they may change at any time, in particular, when new op- + timizations are introduced. If a pattern is too big, a call to pcre2_jit_compile() returns PCRE2_ERROR_NOMEMORY. PCRE2_JIT_COMPLETE requests the JIT compiler to generate code for com- @@ -4987,8 +5163,8 @@ SIMPLE USE OF JIT the entire compiled pattern is freed by calling pcre2_code_free(). In some circumstances you may need to call additional functions. These - are described in the section entitled "Controlling the JIT stack" - below. + are described in the section entitled "Controlling the JIT stack" be- + low. There are some pcre2_match() options that are not supported by JIT, and there are also some pattern items that JIT cannot handle. Details are @@ -5003,8 +5179,8 @@ SIMPLE USE OF JIT If the JIT compiler finds an unsupported item, no JIT data is gener- ated. You can find out if JIT matching is available after compiling a - pattern by calling pcre2_pattern_info() with the PCRE2_INFO_JITSIZE - option. A non-zero result means that JIT compilation was successful. A + pattern by calling pcre2_pattern_info() with the PCRE2_INFO_JITSIZE op- + tion. A non-zero result means that JIT compilation was successful. A result of 0 means that JIT support is not available, or the pattern was not processed by pcre2_jit_compile(), or the JIT compiler was not able to handle the pattern. @@ -5012,25 +5188,27 @@ SIMPLE USE OF JIT MATCHING SUBJECTS CONTAINING INVALID UTF - When a pattern is compiled with the PCRE2_UTF option, the interpretive - matching function expects its subject string to be a valid sequence of - UTF code units. If it is not, the result is undefined. This is also - true by default of matching via JIT. However, if the option - PCRE2_JIT_INVALID_UTF is passed to pcre2_jit_compile(), code that can - process a subject containing invalid UTF is compiled. - - In this mode, an invalid code unit sequence never matches any pattern - item. It does not match dot, it does not match \p{Any}, it does not - even match negative items such as [^X]. A lookbehind assertion fails if - it encounters an invalid sequence while moving the current point back- - wards. In other words, an invalid UTF code unit sequence acts as a bar- - rier which no match can cross. Reaching an invalid sequence causes an - immediate backtrack. - - Using this option, an application can run matches in arbitrary data, - knowing that any matched strings that are returned will be valid UTF. - This can be useful when searching for text in executable or other - binary files. + When a pattern is compiled with the PCRE2_UTF option, subject strings + are normally expected to be a valid sequence of UTF code units. By de- + fault, this is checked at the start of matching and an error is gener- + ated if invalid UTF is detected. The PCRE2_NO_UTF_CHECK option can be + passed to pcre2_match() to skip the check (for improved performance) if + you are sure that a subject string is valid. If this option is used + with an invalid string, the result is undefined. + + However, a way of running matches on strings that may contain invalid + UTF sequences is available. Calling pcre2_compile() with the + PCRE2_MATCH_INVALID_UTF option has two effects: it tells the inter- + preter in pcre2_match() to support invalid UTF, and, if pcre2_jit_com- + pile() is called, the compiled JIT code also supports invalid UTF. De- + tails of how this support works, in both the JIT and the interpretive + cases, is given in the pcre2unicode documentation. + + There is also an obsolete option for pcre2_jit_compile() called + PCRE2_JIT_INVALID_UTF, which currently exists only for backward compat- + ibility. It is superseded by the pcre2_compile() option + PCRE2_MATCH_INVALID_UTF and should no longer be used. It may be removed + in future. UNSUPPORTED OPTIONS AND PATTERN ITEMS @@ -5068,11 +5246,11 @@ CONTROLLING THE JIT STACK When the compiled JIT code runs, it needs a block of memory to use as a stack. By default, it uses 32KiB on the machine stack. However, some - large or complicated patterns need more than this. The error - PCRE2_ERROR_JIT_STACKLIMIT is given when there is not enough stack. - Three functions are provided for managing blocks of memory for use as - JIT stacks. There is further discussion about the use of JIT stacks in - the section entitled "JIT stack FAQ" below. + large or complicated patterns need more than this. The error PCRE2_ER- + ROR_JIT_STACKLIMIT is given when there is not enough stack. Three func- + tions are provided for managing blocks of memory for use as JIT stacks. + There is further discussion about the use of JIT stacks in the section + entitled "JIT stack FAQ" below. The pcre2_jit_stack_create() function creates a JIT stack. Its argu- ments are a starting size, a maximum size, and a general context (for @@ -5116,8 +5294,8 @@ CONTROLLING THE JIT STACK A callback function is obeyed whenever JIT code is about to be run; it is not obeyed when pcre2_match() is called with options that are incom- patible for JIT matching. A callback function can therefore be used to - determine whether a match operation was executed by JIT or by the - interpreter. + determine whether a match operation was executed by JIT or by the in- + terpreter. You may safely use the same JIT stack for more than one pattern (either by assigning directly or by callback), as long as the patterns are @@ -5127,10 +5305,10 @@ CONTROLLING THE JIT STACK stack to the one used for currently suspended match(es). In a multithread application, if you do not specify a JIT stack, or if - you assign or pass back NULL from a callback, that is thread-safe, - because each thread has its own machine stack. However, if you assign - or pass back a non-NULL JIT stack, this must be a different stack for - each thread so that the application is thread-safe. + you assign or pass back NULL from a callback, that is thread-safe, be- + cause each thread has its own machine stack. However, if you assign or + pass back a non-NULL JIT stack, this must be a different stack for each + thread so that the application is thread-safe. Strictly speaking, even more is allowed. You can assign the same non- NULL stack to a match context that is used by any number of patterns, @@ -5169,11 +5347,11 @@ JIT STACK FAQ (2) Why don't we simply allocate blocks of memory with malloc()? - Modern operating systems have a nice feature: they can reserve an - address space instead of allocating memory. We can safely allocate mem- - ory pages inside this address space, so the stack could grow without - moving memory data (this is important because of pointers). Thus we can - allocate 1MiB address space, and use only a single memory page (usually + Modern operating systems have a nice feature: they can reserve an ad- + dress space instead of allocating memory. We can safely allocate memory + pages inside this address space, so the stack could grow without moving + memory data (this is important because of pointers). Thus we can allo- + cate 1MiB address space, and use only a single memory page (usually 4KiB) if that is enough. However, we can still grow up to 1MiB anytime if needed. @@ -5193,7 +5371,7 @@ JIT STACK FAQ pcre2_match() again. When you assign the stack to a match context, only a pointer is set. There is no reference counting or any other magic. You can free compiled patterns, contexts, and stacks in any order, any- - time. Just do not call pcre2_match() with a match context pointing to + time. Just do not call pcre2_match() with a match context pointing to an already freed stack, as that will cause SEGFAULT. (Also, do not free a stack currently used by pcre2_match() in another thread). You can also replace the stack in a context at any time when it is not in use. @@ -5276,12 +5454,12 @@ JIT FAST PATH API matching directly instead of calling pcre2_match() (obviously only for patterns that have been successfully processed by pcre2_jit_compile()). - The fast path function is called pcre2_jit_match(), and it takes - exactly the same arguments as pcre2_match(). However, the subject - string must be specified with a length; PCRE2_ZERO_TERMINATED is not - supported. Unsupported option bits (for example, PCRE2_ANCHORED, - PCRE2_ENDANCHORED and PCRE2_COPY_MATCHED_SUBJECT) are ignored, as is - the PCRE2_NO_JIT option. The return values are also the same as for + The fast path function is called pcre2_jit_match(), and it takes ex- + actly the same arguments as pcre2_match(). However, the subject string + must be specified with a length; PCRE2_ZERO_TERMINATED is not sup- + ported. Unsupported option bits (for example, PCRE2_ANCHORED, PCRE2_EN- + DANCHORED and PCRE2_COPY_MATCHED_SUBJECT) are ignored, as is the + PCRE2_NO_JIT option. The return values are also the same as for pcre2_match(), plus PCRE2_ERROR_JIT_BADOPTION if a matching mode (par- tial or complete) is requested that was not compiled. @@ -5310,7 +5488,7 @@ AUTHOR REVISION - Last updated: 06 March 2019 + Last updated: 23 May 2019 Copyright (c) 1997-2019 University of Cambridge. ------------------------------------------------------------------------------ @@ -5329,24 +5507,24 @@ SIZE AND OTHER LIMITATIONS The maximum size of a compiled pattern is approximately 64 thousand code units for the 8-bit and 16-bit libraries if PCRE2 is compiled with - the default internal linkage size, which is 2 bytes for these - libraries. If you want to process regular expressions that are truly + the default internal linkage size, which is 2 bytes for these li- + braries. If you want to process regular expressions that are truly enormous, you can compile PCRE2 with an internal linkage size of 3 or 4 (when building the 16-bit library, 3 is rounded up to 4). See the README file in the source distribution and the pcre2build documentation for details. In these cases the limit is substantially larger. How- - ever, the speed of execution is slower. In the 32-bit library, the - internal linkage size is always 4. + ever, the speed of execution is slower. In the 32-bit library, the in- + ternal linkage size is always 4. The maximum length of a source pattern string is essentially unlimited; it is the largest number a PCRE2_SIZE variable can hold. However, the program that calls pcre2_compile() can specify a smaller limit. The maximum length (in code units) of a subject string is one less than - the largest number a PCRE2_SIZE variable can hold. PCRE2_SIZE is an - unsigned integer type, usually defined as size_t. Its maximum value - (that is ~(PCRE2_SIZE)0) is reserved as a special indicator for zero- - terminated strings and unset offsets. + the largest number a PCRE2_SIZE variable can hold. PCRE2_SIZE is an un- + signed integer type, usually defined as size_t. Its maximum value (that + is ~(PCRE2_SIZE)0) is reserved as a special indicator for zero-termi- + nated strings and unset offsets. All values in repeating quantifiers must be less than 65536. @@ -5404,8 +5582,8 @@ PCRE2 MATCHING ALGORITHMS An alternative algorithm is provided by the pcre2_dfa_match() function; it operates in a different way, and is not Perl-compatible. This alter- - native has advantages and disadvantages compared with the standard - algorithm, and these are described below. + native has advantages and disadvantages compared with the standard al- + gorithm, and these are described below. When there is only one possible way in which a given subject string can match a pattern, the two algorithms give the same answer. A difference @@ -5473,8 +5651,8 @@ THE ALTERNATIVE MATCHING ALGORITHM Although the general principle of this matching algorithm is that it scans the subject string only once, without backtracking, there is one exception: when a lookaround assertion is encountered, the characters - following or preceding the current point have to be independently - inspected. + following or preceding the current point have to be independently in- + spected. The scan continues until either the end of the subject is reached, or there are no more unterminated paths. At this point, terminated paths @@ -5508,9 +5686,9 @@ THE ALTERNATIVE MATCHING ALGORITHM not supported or behave differently in the alternative matching func- tion. Those that are not supported cause an error if encountered. - 1. Because the algorithm finds all possible matches, the greedy or - ungreedy nature of repetition quantifiers is not relevant (though it - may affect auto-possessification, as just described). During matching, + 1. Because the algorithm finds all possible matches, the greedy or un- + greedy nature of repetition quantifiers is not relevant (though it may + affect auto-possessification, as just described). During matching, greedy and ungreedy quantifiers are treated in exactly the same way. However, possessive quantifiers can make a difference when what follows could also match what is quantified, for example in a pattern like @@ -5539,8 +5717,8 @@ THE ALTERNATIVE MATCHING ALGORITHM 5. Again for the same reason, script runs are not supported. - 6. Because many paths through the tree may be active, the \K escape - sequence, which resets the start of the match when encountered (but may + 6. Because many paths through the tree may be active, the \K escape se- + quence, which resets the start of the match when encountered (but may be on some paths and not on others), is not supported. 7. Callouts are supported, but the value of the capture_top field is @@ -5556,23 +5734,26 @@ THE ALTERNATIVE MATCHING ALGORITHM are not supported. (*FAIL) is supported, and behaves like a failing negative assertion. + 10. The PCRE2_MATCH_INVALID_UTF option for pcre2_compile() is not sup- + ported by pcre2_dfa_match(). + ADVANTAGES OF THE ALTERNATIVE ALGORITHM - Using the alternative matching algorithm provides the following advan- + Using the alternative matching algorithm provides the following advan- tages: 1. All possible matches (at a single point in the subject) are automat- - ically found, and in particular, the longest match is found. To find + ically found, and in particular, the longest match is found. To find more than one match using the standard algorithm, you have to do kludgy things with callouts. - 2. Because the alternative algorithm scans the subject string just + 2. Because the alternative algorithm scans the subject string just once, and never needs to backtrack (except for lookbehinds), it is pos- - sible to pass very long subject strings to the matching function in + sible to pass very long subject strings to the matching function in several pieces, checking for partial matching each time. Although it is - also possible to do multi-segment matching using the standard algo- - rithm, by retaining partially matched substrings, it is more compli- + also possible to do multi-segment matching using the standard algo- + rithm, by retaining partially matched substrings, it is more compli- cated. The pcre2partial documentation gives details of partial matching and discusses multi-segment matching. @@ -5581,12 +5762,12 @@ DISADVANTAGES OF THE ALTERNATIVE ALGORITHM The alternative algorithm suffers from a number of disadvantages: - 1. It is substantially slower than the standard algorithm. This is - partly because it has to search for all possible matches, but is also + 1. It is substantially slower than the standard algorithm. This is + partly because it has to search for all possible matches, but is also because it is less susceptible to optimization. - 2. Capturing parentheses, backreferences, and script runs are not sup- - ported. + 2. Capturing parentheses, backreferences, script runs, and matching + within invalid UTF string are not supported. 3. Although atomic groups are supported, their use does not provide the performance advantage that it does for the standard algorithm. @@ -5601,8 +5782,8 @@ AUTHOR REVISION - Last updated: 10 October 2018 - Copyright (c) 1997-2018 University of Cambridge. + Last updated: 23 May 2019 + Copyright (c) 1997-2019 University of Cambridge. ------------------------------------------------------------------------------ @@ -5615,66 +5796,109 @@ NAME PARTIAL MATCHING IN PCRE2 - In normal use of PCRE2, if the subject string that is passed to a - matching function matches as far as it goes, but is too short to match - the entire pattern, PCRE2_ERROR_NOMATCH is returned. There are circum- - stances where it might be helpful to distinguish this case from other - cases in which there is no match. - - Consider, for example, an application where a human is required to type - in data for a field with specific formatting requirements. An example - might be a date in the form ddmmmyy, defined by this pattern: - - ^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$ - - If the application sees the user's keystrokes one by one, and can check - that what has been typed so far is potentially valid, it is able to - raise an error as soon as a mistake is made, by beeping and not - reflecting the character that has been typed, for example. This immedi- - ate feedback is likely to be a better user interface than a check that - is delayed until the entire string has been entered. Partial matching - can also be useful when the subject string is very long and is not all - available at once. - - PCRE2 supports partial matching by means of the PCRE2_PARTIAL_SOFT and - PCRE2_PARTIAL_HARD options, which can be set when calling a matching - function. The difference between the two options is whether or not a - partial match is preferred to an alternative complete match, though the - details differ between the two types of matching function. If both - options are set, PCRE2_PARTIAL_HARD takes precedence. - - If you want to use partial matching with just-in-time optimized code, - you must call pcre2_jit_compile() with one or both of these options: + In normal use of PCRE2, if there is a match up to the end of a subject + string, but more characters are needed to match the entire pattern, + PCRE2_ERROR_NOMATCH is returned, just like any other failing match. + There are circumstances where it might be helpful to distinguish this + "partial match" case. + + One example is an application where the subject string is very long, + and not all available at once. The requirement here is to be able to do + the matching segment by segment, but special action is needed when a + matched substring spans the boundary between two segments. + + Another example is checking a user input string as it is typed, to en- + sure that it conforms to a required format. Invalid characters can be + immediately diagnosed and rejected, giving instant feedback. + + Partial matching is a PCRE2-specific feature; it is not Perl-compati- + ble. It is requested by setting one of the PCRE2_PARTIAL_HARD or + PCRE2_PARTIAL_SOFT options when calling a matching function. The dif- + ference between the two options is whether or not a partial match is + preferred to an alternative complete match, though the details differ + between the two types of matching function. If both options are set, + PCRE2_PARTIAL_HARD takes precedence. + + If you want to use partial matching with just-in-time optimized code, + as well as setting a partial match option for the matching function, + you must also call pcre2_jit_compile() with one or both of these op- + tions: - PCRE2_JIT_PARTIAL_SOFT PCRE2_JIT_PARTIAL_HARD + PCRE2_JIT_PARTIAL_SOFT + + PCRE2_JIT_COMPLETE should also be set if you are going to run non-par- + tial matches on the same pattern. Separate code is compiled for each + mode. If the appropriate JIT mode has not been compiled, interpretive + matching code is used. + + Setting a partial matching option disables two of PCRE2's standard op- + timization hints. PCRE2 remembers the last literal code unit in a pat- + tern, and abandons matching immediately if it is not present in the + subject string. This optimization cannot be used for a subject string + that might match only partially. PCRE2 also remembers a minimum length + of a matching string, and does not bother to run the matching function + on shorter strings. This optimization is also disabled for partial + matching. + + +REQUIREMENTS FOR A PARTIAL MATCH + + A possible partial match occurs during matching when the end of the + subject string is reached successfully, but either more characters are + needed to complete the match, or the addition of more characters might + change what is matched. + + Example 1: if the pattern is /abc/ and the subject is "ab", more char- + acters are definitely needed to complete a match. In this case both + hard and soft matching options yield a partial match. - PCRE2_JIT_COMPLETE should also be set if you are going to run non-par- - tial matches on the same pattern. If the appropriate JIT mode has not - been compiled, interpretive matching code is used. + Example 2: if the pattern is /ab+/ and the subject is "ab", a complete + match can be found, but the addition of more characters might change + what is matched. In this case, only PCRE2_PARTIAL_HARD returns a par- + tial match; PCRE2_PARTIAL_SOFT returns the complete match. - Setting a partial matching option disables two of PCRE2's standard - optimizations. PCRE2 remembers the last literal code unit in a pattern, - and abandons matching immediately if it is not present in the subject - string. This optimization cannot be used for a subject string that - might match only partially. PCRE2 also knows the minimum length of a - matching string, and does not bother to run the matching function on - shorter strings. This optimization is also disabled for partial match- - ing. + On reaching the end of the subject, when PCRE2_PARTIAL_HARD is set, if + the next pattern item is \z, \Z, \b, \B, or $ there is always a partial + match. Otherwise, for both options, the next pattern item must be one + that inspects a character, and at least one of the following must be + true: + + (1) At least one character has already been inspected. An inspected + character need not form part of the final matched string; lookbehind + assertions and the \K escape sequence provide ways of inspecting char- + acters before the start of a matched string. + + (2) The pattern contains one or more lookbehind assertions. This condi- + tion exists in case there is a lookbehind that inspects characters be- + fore the start of the match. + + (3) There is a special case when the whole pattern can match an empty + string. When the starting point is at the end of the subject, the + empty string match is a possibility, and if PCRE2_PARTIAL_SOFT is set + and neither of the above conditions is true, it is returned. However, + because adding more characters might result in a non-empty match, + PCRE2_PARTIAL_HARD returns a partial match, which in this case means + "there is going to be a match at this point, but until some more char- + acters are added, we do not know if it will be an empty string or some- + thing longer". PARTIAL MATCHING USING pcre2_match() - A partial match occurs during a call to pcre2_match() when the end of - the subject string is reached successfully, but matching cannot con- - tinue because more characters are needed. However, at least one charac- - ter in the subject must have been inspected. This character need not - form part of the final matched string; lookbehind assertions and the \K - escape sequence provide ways of inspecting characters before the start - of a matched string. The requirement for inspecting at least one char- - acter exists because an empty string can always be matched; without - such a restriction there would always be a partial match of an empty - string at the end of the subject. + When a partial matching option is set, the result of calling + pcre2_match() can be one of the following: + + A successful match + A complete match has been found, starting and ending within this sub- + ject. + + PCRE2_ERROR_NOMATCH + No match can start anywhere in this subject. + + PCRE2_ERROR_PARTIAL + Adding more characters may result in a complete match that uses one + or more characters from the end of this subject. When a partial match is returned, the first two elements in the ovector point to the portion of the subject that was matched, but the values in @@ -5684,29 +5908,12 @@ PARTIAL MATCHING USING pcre2_match() /abc\K123/ If it is matched against "456abc123xyz" the result is a complete match, - and the ovector defines the matched string as "123", because \K resets - the "start of match" point. However, if a partial match is requested - and the subject string is "456abc12", a partial match is found for the - string "abc12", because all these characters are needed for a subse- + and the ovector defines the matched string as "123", because \K resets + the "start of match" point. However, if a partial match is requested + and the subject string is "456abc12", a partial match is found for the + string "abc12", because all these characters are needed for a subse- quent re-match with additional characters. - What happens when a partial match is identified depends on which of the - two partial matching options are set. - - PCRE2_PARTIAL_SOFT WITH pcre2_match() - - If PCRE2_PARTIAL_SOFT is set when pcre2_match() identifies a partial - match, the partial match is remembered, but matching continues as nor- - mal, and other alternatives in the pattern are tried. If no complete - match can be found, PCRE2_ERROR_PARTIAL is returned instead of - PCRE2_ERROR_NOMATCH. - - This option is "soft" because it prefers a complete match over a par- - tial match. All the various matching items in a pattern behave as if - the subject string is potentially complete. For example, \z, \Z, and $ - match at the end of the subject, as normal, and for \b and \B the end - of the subject is treated as a non-alphanumeric. - If there is more than one partial match, the first one that was found provides the data that is returned. Consider this pattern: @@ -5715,23 +5922,31 @@ PARTIAL MATCHING USING pcre2_match() If this is matched against the subject string "abc123dog", both alter- natives fail to match, but the end of the subject is reached during matching, so PCRE2_ERROR_PARTIAL is returned. The offsets are set to 3 - and 9, identifying "123dog" as the first partial match that was found. - (In this example, there are two partial matches, because "dog" on its - own partially matches the second alternative.) - - PCRE2_PARTIAL_HARD WITH pcre2_match() + and 9, identifying "123dog" as the first partial match. (In this exam- + ple, there are two partial matches, because "dog" on its own partially + matches the second alternative.) - If PCRE2_PARTIAL_HARD is set for pcre2_match(), PCRE2_ERROR_PARTIAL is - returned as soon as a partial match is found, without continuing to - search for possible complete matches. This option is "hard" because it - prefers an earlier partial match over a later complete match. For this - reason, the assumption is made that the end of the supplied subject - string may not be the true end of the available data, and so, if \z, - \Z, \b, \B, or $ are encountered at the end of the subject, the result - is PCRE2_ERROR_PARTIAL, provided that at least one character in the - subject has been inspected. + How a partial match is processed by pcre2_match() - Comparing hard and soft partial matching + What happens when a partial match is identified depends on which of the + two partial matching options is set. + + If PCRE2_PARTIAL_HARD is set, PCRE2_ERROR_PARTIAL is returned as soon + as a partial match is found, without continuing to search for possible + complete matches. This option is "hard" because it prefers an earlier + partial match over a later complete match. For this reason, the assump- + tion is made that the end of the supplied subject string is not the + true end of the available data, which is why \z, \Z, \b, \B, and $ al- + ways give a partial match. + + If PCRE2_PARTIAL_SOFT is set, the partial match is remembered, but + matching continues as normal, and other alternatives in the pattern are + tried. If no complete match can be found, PCRE2_ERROR_PARTIAL is re- + turned instead of PCRE2_ERROR_NOMATCH. This option is "soft" because it + prefers a complete match over a partial match. All the various matching + items in a pattern behave as if the subject string is potentially com- + plete; \z, \Z, and $ match at the end of the subject, as normal, and + for \b and \B the end of the subject is treated as a non-alphanumeric. The difference between the two partial matching options can be illus- trated by a pattern such as: @@ -5758,89 +5973,150 @@ PARTIAL MATCHING USING pcre2_match() The second pattern will never match "dogsbody", because it will always find the shorter match first. + Example of partial matching using pcre2test -PARTIAL MATCHING USING pcre2_dfa_match() + The pcre2test data modifiers partial_hard (or ph) and partial_soft (or + ps) set PCRE2_PARTIAL_HARD and PCRE2_PARTIAL_SOFT, respectively, when + calling pcre2_match(). Here is a run of pcre2test using a pattern that + matches the whole subject in the form of a date: - The DFA functions move along the subject string character by character, - without backtracking, searching for all possible matches simultane- - ously. If the end of the subject is reached before the end of the pat- - tern, there is the possibility of a partial match, again provided that - at least one character has been inspected. + re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ + data> 25dec3\=ph + Partial match: 23dec3 + data> 3ju\=ph + Partial match: 3ju + data> 3juj\=ph + No match - When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if - there have been no complete matches. Otherwise, the complete matches - are returned. However, if PCRE2_PARTIAL_HARD is set, a partial match - takes precedence over any complete matches. The portion of the string - that was matched when the longest partial match was found is set as the - first matching string. + This example gives the same results for both hard and soft partial + matching options. Here is an example where there is a difference: - Because the DFA functions always search for all possible matches, and - there is no difference between greedy and ungreedy repetition, their - behaviour is different from the standard functions when PCRE2_PAR- - TIAL_HARD is set. Consider the string "dog" matched against the - ungreedy pattern shown above: + re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ + data> 25jun04\=ps + 0: 25jun04 + 1: jun + data> 25jun04\=ph + Partial match: 25jun04 - /dog(sbody)??/ + With PCRE2_PARTIAL_SOFT, the subject is matched completely. For + PCRE2_PARTIAL_HARD, however, the subject is assumed not to be complete, + so there is only a partial match. - Whereas the standard function stops as soon as it finds the complete - match for "dog", the DFA function also finds the partial match for - "dogsbody", and so returns that when PCRE2_PARTIAL_HARD is set. +MULTI-SEGMENT MATCHING WITH pcre2_match() -PARTIAL MATCHING AND WORD BOUNDARIES + PCRE was not originally designed with multi-segment matching in mind. + However, over time, features (including partial matching) that make + multi-segment matching possible have been added. A very long string can + be searched segment by segment by calling pcre2_match() repeatedly, + with the aim of achieving the same results that would happen if the en- + tire string was available for searching all the time. Normally, the + strings that are being sought are much shorter than each individual + segment, and are in the middle of very long strings, so the pattern is + normally not anchored. + + Special logic must be implemented to handle a matched substring that + spans a segment boundary. PCRE2_PARTIAL_HARD should be used, because it + returns a partial match at the end of a segment whenever there is the + possibility of changing the match by adding more characters. The + PCRE2_NOTBOL option should also be set for all but the first segment. + + When a partial match occurs, the next segment must be added to the cur- + rent subject and the match re-run, using the startoffset argument of + pcre2_match() to begin at the point where the partial match started. + For example: - If a pattern ends with one of sequences \b or \B, which test for word - boundaries, partial matching with PCRE2_PARTIAL_SOFT can give counter- - intuitive results. Consider this pattern: + re> /\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d/ + data> ...the date is 23ja\=ph + Partial match: 23ja + data> ...the date is 23jan19 and on that day...\=offset=15 + 0: 23jan19 + 1: jan + + Note the use of the offset modifier to start the new match where the + partial match was found. In this example, the next segment was added to + the one in which the partial match was found. This is the most + straightforward approach, typically using a memory buffer that is twice + the size of each segment. After a partial match, the first half of the + buffer is discarded, the second half is moved to the start of the buf- + fer, and a new segment is added before repeating the match as in the + example above. After a no match, the entire buffer can be discarded. + + If there are memory constraints, you may want to discard text that pre- + cedes a partial match before adding the next segment. Unfortunately, + this is not at present straightforward. In cases such as the above, + where the pattern does not contain any lookbehinds, it is sufficient to + retain only the partially matched substring. However, if the pattern + contains a lookbehind assertion, characters that precede the start of + the partial match may have been inspected during the matching process. + When pcre2test displays a partial match, it indicates these characters + with '<' if the allusedtext modifier is set: - /\bcat\b/ + re> "(?<=123)abc" + data> xx123ab\=ph,allusedtext + Partial match: 123ab + <<< - This matches "cat", provided there is a word boundary at either end. If - the subject string is "the cat", the comparison of the final "t" with a - following character cannot take place, so a partial match is found. - However, normal matching carries on, and \b matches at the end of the - subject when the last character is a letter, so a complete match is - found. The result, therefore, is not PCRE2_ERROR_PARTIAL. Using - PCRE2_PARTIAL_HARD in this case does yield PCRE2_ERROR_PARTIAL, because - then the partial match takes precedence. + However, the allusedtext modifier is not available for JIT matching, + because JIT matching does not record the first (or last) consulted + characters. For this reason, this information is not available via the + API. It is therefore not possible in general to obtain the exact number + of characters that must be retained in order to get the right match re- + sult. If you cannot retain the entire segment, you must find some + heuristic way of choosing. + + If you know the approximate length of the matching substrings, you can + use that to decide how much text to retain. The only lookbehind infor- + mation that is currently available via the API is the length of the + longest individual lookbehind in a pattern, but this can be misleading + if there are nested lookbehinds. The value returned by calling + pcre2_pattern_info() with the PCRE2_INFO_MAXLOOKBEHIND option is the + maximum number of characters (not code units) that any individual look- + behind moves back when it is processed. A pattern such as + "(?<=(?<!b)a)" has a maximum lookbehind value of one, but inspects two + characters before its starting point. + + In a non-UTF or a 32-bit case, moving back is just a subtraction, but + in UTF-8 or UTF-16 you have to count characters while moving back + through the code units. -EXAMPLE OF PARTIAL MATCHING USING PCRE2TEST +PARTIAL MATCHING USING pcre2_dfa_match() - If the partial_soft (or ps) modifier is present on a pcre2test data - line, the PCRE2_PARTIAL_SOFT option is used for the match. Here is a - run of pcre2test that uses the date example quoted above: + The DFA function moves along the subject string character by character, + without backtracking, searching for all possible matches simultane- + ously. If the end of the subject is reached before the end of the pat- + tern, there is the possibility of a partial match. - re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ - data> 25jun04\=ps - 0: 25jun04 - 1: jun - data> 25dec3\=ps - Partial match: 23dec3 - data> 3ju\=ps - Partial match: 3ju - data> 3juj\=ps - No match - data> j\=ps - No match + When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if + there have been no complete matches. Otherwise, the complete matches + are returned. If PCRE2_PARTIAL_HARD is set, a partial match takes + precedence over any complete matches. The portion of the string that + was matched when the longest partial match was found is set as the + first matching string. - The first data string is matched completely, so pcre2test shows the - matched substrings. The remaining four strings do not match the com- - plete pattern, but the first two are partial matches. Similar output is - obtained if DFA matching is used. + Because the DFA function always searches for all possible matches, and + there is no difference between greedy and ungreedy repetition, its be- + haviour is different from the pcre2_match(). Consider the string "dog" + matched against this ungreedy pattern: - If the partial_hard (or ph) modifier is present on a pcre2test data - line, the PCRE2_PARTIAL_HARD option is set for the match. + /dog(sbody)??/ + + Whereas the standard function stops as soon as it finds the complete + match for "dog", the DFA function also finds the partial match for + "dogsbody", and so returns that when PCRE2_PARTIAL_HARD is set. MULTI-SEGMENT MATCHING WITH pcre2_dfa_match() - When a partial match has been found using a DFA matching function, it + When a partial match has been found using the DFA matching function, it is possible to continue the match by providing additional subject data and calling the function again with the same compiled regular expres- sion, this time setting the PCRE2_DFA_RESTART option. You must pass the same working space as before, because this is where details of the pre- - vious partial match are stored. Here is an example using pcre2test: + vious partial match are stored. You can set the PCRE2_PARTIAL_SOFT or + PCRE2_PARTIAL_HARD options with PCRE2_DFA_RESTART to continue partial + matching over multiple segments. Here is an example using pcre2test: re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ data> 23ja\=dfa,ps @@ -5853,152 +6129,10 @@ MULTI-SEGMENT MATCHING WITH pcre2_dfa_match() (restarted) match. Notice that when the match is complete, only the last part is shown; PCRE2 does not retain the previously partially- matched string. It is up to the calling program to do that if it needs - to. - - That means that, for an unanchored pattern, if a continued match fails, - it is not possible to try again at a new starting point. All this - facility is capable of doing is continuing with the previous match - attempt. In the previous example, if the second set of data is "ug23" - the result is no match, even though there would be a match for "aug23" - if the entire string were given at once. Depending on the application, - this may or may not be what you want. The only way to allow for start- - ing again at the next character is to retain the matched part of the - subject and try a new complete match. - - You can set the PCRE2_PARTIAL_SOFT or PCRE2_PARTIAL_HARD options with - PCRE2_DFA_RESTART to continue partial matching over multiple segments. - This facility can be used to pass very long subject strings to the DFA - matching functions. - - -MULTI-SEGMENT MATCHING WITH pcre2_match() - - Unlike the DFA function, it is not possible to restart the previous - match with a new segment of data when using pcre2_match(). Instead, new - data must be added to the previous subject string, and the entire match - re-run, starting from the point where the partial match occurred. Ear- - lier data can be discarded. - - It is best to use PCRE2_PARTIAL_HARD in this situation, because it does - not treat the end of a segment as the end of the subject when matching - \z, \Z, \b, \B, and $. Consider an unanchored pattern that matches - dates: - - re> /\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d/ - data> The date is 23ja\=ph - Partial match: 23ja - - At this stage, an application could discard the text preceding "23ja", - add on text from the next segment, and call the matching function - again. Unlike the DFA matching function, the entire matching string - must always be available, and the complete matching process occurs for - each call, so more memory and more processing time is needed. - - -ISSUES WITH MULTI-SEGMENT MATCHING - - Certain types of pattern may give problems with multi-segment matching, - whichever matching function is used. - - 1. If the pattern contains a test for the beginning of a line, you need - to pass the PCRE2_NOTBOL option when the subject string for any call - does start at the beginning of a line. There is also a PCRE2_NOTEOL - option, but in practice when doing multi-segment matching you should be - using PCRE2_PARTIAL_HARD, which includes the effect of PCRE2_NOTEOL. - - 2. If a pattern contains a lookbehind assertion, characters that pre- - cede the start of the partial match may have been inspected during the - matching process. When using pcre2_match(), sufficient characters must - be retained for the next match attempt. You can ensure that enough - characters are retained by doing the following: - - Before doing any matching, find the length of the longest lookbehind in - the pattern by calling pcre2_pattern_info() with the - PCRE2_INFO_MAXLOOKBEHIND option. Note that the resulting count is in - characters, not code units. After a partial match, moving back from the - ovector[0] offset in the subject by the number of characters given for - the maximum lookbehind gets you to the earliest character that must be - retained. In a non-UTF or a 32-bit situation, moving back is just a - subtraction, but in UTF-8 or UTF-16 you have to count characters while - moving back through the code units. - - Characters before the point you have now reached can be discarded, and - after the next segment has been added to what is retained, you should - run the next match with the startoffset argument set so that the match - begins at the same point as before. - - For example, if the pattern "(?<=123)abc" is partially matched against - the string "xx123ab", the ovector offsets are 5 and 7 ("ab"). The maxi- - mum lookbehind count is 3, so all characters before offset 2 can be - discarded. The value of startoffset for the next match should be 3. - When pcre2test displays a partial match, it indicates the lookbehind - characters with '<' characters: - - re> "(?<=123)abc" - data> xx123ab\=ph - Partial match: 123ab - <<< - - 3. Because a partial match must always contain at least one character, - what might be considered a partial match of an empty string actually - gives a "no match" result. For example: - - re> /c(?<=abc)x/ - data> ab\=ps - No match - - If the next segment begins "cx", a match should be found, but this will - only happen if characters from the previous segment are retained. For - this reason, a "no match" result should be interpreted as "partial - match of an empty string" when the pattern contains lookbehinds. - - 4. Matching a subject string that is split into multiple segments may - not always produce exactly the same result as matching over one single - long string, especially when PCRE2_PARTIAL_SOFT is used. The section - "Partial Matching and Word Boundaries" above describes an issue that - arises if the pattern ends with \b or \B. Another kind of difference - may occur when there are multiple matching possibilities, because (for - PCRE2_PARTIAL_SOFT) a partial match result is given only when there are - no completed matches. This means that as soon as the shortest match has - been found, continuation to a new subject segment is no longer possi- - ble. Consider this pcre2test example: - - re> /dog(sbody)?/ - data> dogsb\=ps - 0: dog - data> do\=ps,dfa - Partial match: do - data> gsb\=ps,dfa,dfa_restart - 0: g - data> dogsbody\=dfa - 0: dogsbody - 1: dog - - The first data line passes the string "dogsb" to a standard matching - function, setting the PCRE2_PARTIAL_SOFT option. Although the string is - a partial match for "dogsbody", the result is not PCRE2_ERROR_PARTIAL, - because the shorter string "dog" is a complete match. Similarly, when - the subject is presented to a DFA matching function in several parts - ("do" and "gsb" being the first two) the match stops when "dog" has - been found, and it is not possible to continue. On the other hand, if - "dogsbody" is presented as a single string, a DFA matching function - finds both matches. - - Because of these problems, it is best to use PCRE2_PARTIAL_HARD when - matching multi-segment data. The example above then behaves differ- - ently: - - re> /dog(sbody)?/ - data> dogsb\=ph - Partial match: dogsb - data> do\=ps,dfa - Partial match: do - data> gsb\=ph,dfa,dfa_restart - Partial match: gsb - - 5. Patterns that contain alternatives at the top level which do not all - start with the same pattern item may not work as expected when - PCRE2_DFA_RESTART is used. For example, consider this pattern: + to. This means that, for an unanchored pattern, if a continued match + fails, it is not possible to try again at a new starting point. All + this facility is capable of doing is continuing with the previous match + attempt. For example, consider this pattern: 1234|3789 @@ -6007,29 +6141,16 @@ ISSUES WITH MULTI-SEGMENT MATCHING the second alternative, because such a match does not start at the same point in the subject string. Attempting to continue with the string "7890" does not yield a match because only those alternatives that - match at one point in the subject are remembered. The problem arises - because the start of the second alternative matches within the first - alternative. There is no problem with anchored patterns or patterns - such as: - - 1234|ABCD + match at one point in the subject are remembered. Depending on the ap- + plication, this may or may not be what you want. - where no string can be a partial match for both alternatives. This is - not a problem if a standard matching function is used, because the - entire match has to be rerun each time: - - re> /1234|3789/ - data> ABC123\=ph - Partial match: 123 - data> 1237890 - 0: 3789 - - Of course, instead of using PCRE2_DFA_RESTART, the same technique of - re-running the entire match can also be used with the DFA matching - function. Another possibility is to work with two buffers. If a partial - match at offset n in the first buffer is followed by "no match" when - PCRE2_DFA_RESTART is used on the second buffer, you can then try a new - match starting at offset n+1 in the first buffer. + If you do want to allow for starting again at the next character, one + way of doing it is to retain some or all of the segment and try a new + complete match, as described for pcre2_match() above. Another possibil- + ity is to work with two buffers. If a partial match at offset n in the + first buffer is followed by "no match" when PCRE2_DFA_RESTART is used + on the second buffer, you can then try a new match starting at offset + n+1 in the first buffer. AUTHOR @@ -6041,8 +6162,8 @@ AUTHOR REVISION - Last updated: 22 December 2014 - Copyright (c) 1997-2014 University of Cambridge. + Last updated: 04 September 2019 + Copyright (c) 1997-2019 University of Cambridge. ------------------------------------------------------------------------------ @@ -6065,10 +6186,10 @@ PCRE2 REGULAR EXPRESSION DETAILS Perl's regular expressions are described in its own documentation, and regular expressions in general are covered in a number of books, some - of which have copious examples. Jeffrey Friedl's "Mastering Regular - Expressions", published by O'Reilly, covers regular expressions in - great detail. This description of PCRE2's regular expressions is - intended as reference material. + of which have copious examples. Jeffrey Friedl's "Mastering Regular Ex- + pressions", published by O'Reilly, covers regular expressions in great + detail. This description of PCRE2's regular expressions is intended as + reference material. This document discusses the regular expression patterns that are sup- ported by PCRE2 when its main matching function, pcre2_match(), is @@ -6086,8 +6207,8 @@ SPECIAL START-OF-PATTERN ITEMS set by special items at the start of a pattern. These are not Perl-com- patible, but are provided to make these options accessible to pattern writers who are not able to change the program that processes the pat- - tern. Any number of these items may appear, but they must all be - together right at the start of the pattern string, and the letters must + tern. Any number of these items may appear, but they must all be to- + gether right at the start of the pattern string, and the letters must be in upper case. UTF support @@ -6098,16 +6219,16 @@ SPECIAL START-OF-PATTERN ITEMS the character values to valid Unicode code points. To process UTF strings, PCRE2 must be built to include Unicode support (which is the default). When using UTF strings you must either call the compiling - function with the PCRE2_UTF option, or the pattern must start with the - special sequence (*UTF), which is equivalent to setting the relevant - option. How setting a UTF mode affects pattern matching is mentioned in - several places below. There is also a summary of features in the - pcre2unicode page. + function with one or both of the PCRE2_UTF or PCRE2_MATCH_INVALID_UTF + options, or the pattern must start with the special sequence (*UTF), + which is equivalent to setting the relevant PCRE2_UTF. How setting a + UTF mode affects pattern matching is mentioned in several places below. + There is also a summary of features in the pcre2unicode page. Some applications that allow their users to supply patterns may wish to restrict them to non-UTF data for security reasons. If the - PCRE2_NEVER_UTF option is passed to pcre2_compile(), (*UTF) is not - allowed, and its appearance in a pattern causes an error. + PCRE2_NEVER_UTF option is passed to pcre2_compile(), (*UTF) is not al- + lowed, and its appearance in a pattern causes an error. Unicode property support @@ -6115,7 +6236,9 @@ SPECIAL START-OF-PATTERN ITEMS (*UCP). This has the same effect as setting the PCRE2_UCP option: it causes sequences such as \d and \w to use Unicode properties to deter- mine character types, instead of recognizing only characters with codes - less than 256 via a lookup table. + less than 256 via a lookup table. If also causes upper/lower casing op- + erations to use Unicode properties for characters with code points + greater than 127, even when UTF is not set. Some applications that allow their users to supply patterns may wish to restrict them for security reasons. If the PCRE2_NEVER_UCP option is @@ -6127,8 +6250,8 @@ SPECIAL START-OF-PATTERN ITEMS Starting a pattern with (*NOTEMPTY) or (*NOTEMPTY_ATSTART) has the same effect as passing the PCRE2_NOTEMPTY or PCRE2_NOTEMPTY_ATSTART option to whichever matching function is subsequently called to match the pat- - tern. These options lock out the matching of empty strings, either - entirely, or only at the start of the subject. + tern. These options lock out the matching of empty strings, either en- + tirely, or only at the start of the subject. Disabling auto-possessification @@ -6220,8 +6343,8 @@ SPECIAL START-OF-PATTERN ITEMS (*NUL) the NUL character (binary zero) These override the default and the options given to the compiling func- - tion. For example, on a Unix system where LF is the default newline - sequence, the pattern + tion. For example, on a Unix system where LF is the default newline se- + quence, the pattern (*CR)a.b @@ -6233,8 +6356,8 @@ SPECIAL START-OF-PATTERN ITEMS tions are true. It also affects the interpretation of the dot metachar- acter when PCRE2_DOTALL is not set, and the behaviour of \N when not followed by an opening brace. However, it does not affect what the \R - escape sequence matches. By default, this is any Unicode newline - sequence, for Perl compatibility. However, this can be changed; see the + escape sequence matches. By default, this is any Unicode newline se- + quence, for Perl compatibility. However, this can be changed; see the next section and the description of \R in the section entitled "Newline sequences" below. A change of \R setting can be combined with a change of newline convention. @@ -6267,8 +6390,12 @@ CHARACTERS AND METACHARACTERS The quick brown fox matches a portion of a subject string that is identical to itself. When - caseless matching is specified (the PCRE2_CASELESS option), letters are - matched independently of case. + caseless matching is specified (the PCRE2_CASELESS option or (?i) + within the pattern), letters are matched independently of case. Note + that there are two ASCII characters, K and S, that, in addition to + their lower case ASCII equivalents, are case-equivalent with Unicode + U+212A (Kelvin sign) and U+017F (long S) respectively when either + PCRE2_UTF or PCRE2_UCP is set. The power of regular expressions comes from the ability to include wild cards, character classes, alternatives, and repetitions in the pattern. @@ -6303,6 +6430,18 @@ CHARACTERS AND METACHARACTERS [ POSIX character class (if followed by POSIX syntax) ] terminates the character class + If a pattern is compiled with the PCRE2_EXTENDED option, most white + space in the pattern, other than in a character class, and characters + between a # outside a character class and the next newline, inclusive, + are ignored. An escaping backslash can be used to include a white space + or a # character as part of the pattern. If the PCRE2_EXTENDED_MORE op- + tion is set, the same applies, but in addition unescaped space and hor- + izontal tab characters are ignored inside a character class. Note: only + these two characters are ignored, not the full set of pattern white + space characters that are ignored outside a character class. Option + settings can be changed within a pattern; see the section entitled "In- + ternal Option Setting" below. + The following sections describe the use of each of the metacharacters. @@ -6320,15 +6459,9 @@ BACKSLASH that it stands for itself. In particular, if you want to match a back- slash, you write \\. - In a UTF mode, only ASCII digits and letters have any special meaning - after a backslash. All other characters (in particular, those whose - code points are greater than 127) are treated as literals. - - If a pattern is compiled with the PCRE2_EXTENDED option, most white - space in the pattern (other than in a character class), and characters - between a # outside a character class and the next newline, inclusive, - are ignored. An escaping backslash can be used to include a white space - or # character as part of the pattern. + Only ASCII digits and letters have any special meaning after a back- + slash. All other characters (in particular, those whose code points are + greater than 127) are treated as literals. If you want to treat all characters in a sequence as literals, you can do so by putting them between \Q and \E. This is different from Perl in @@ -6392,72 +6525,71 @@ BACKSLASH as \x{dc} or \334. However, using the braced versions does make such sequences easier to read. - Support is available for some ECMAScript (aka JavaScript) escape - sequences via two compile-time options. If PCRE2_ALT_BSUX is set, the - sequence \x followed by { is not recognized. Only if \x is followed by + Support is available for some ECMAScript (aka JavaScript) escape se- + quences via two compile-time options. If PCRE2_ALT_BSUX is set, the se- + quence \x followed by { is not recognized. Only if \x is followed by two hexadecimal digits is it recognized as a character escape. Other- wise it is interpreted as a literal "x" character. In this mode, sup- port for code points greater than 256 is provided by \u, which must be followed by four hexadecimal digits; otherwise it is interpreted as a literal "u" character. - PCRE2_EXTRA_ALT_BSUX has the same effect as PCRE2_ALT_BSUX and, in - addition, \u{hhh..} is recognized as the character specified by hexa- - decimal code point. There may be any number of hexadecimal digits. - This syntax is from ECMAScript 6. + PCRE2_EXTRA_ALT_BSUX has the same effect as PCRE2_ALT_BSUX and, in ad- + dition, \u{hhh..} is recognized as the character specified by hexadeci- + mal code point. There may be any number of hexadecimal digits. This + syntax is from ECMAScript 6. - The \N{U+hhh..} escape sequence is recognized only when the PCRE2_UTF - option is set, that is, when PCRE2 is operating in a Unicode mode. Perl - also uses \N{name} to specify characters by Unicode name; PCRE2 does - not support this. Note that when \N is not followed by an opening - brace (curly bracket) it has an entirely different meaning, matching - any character that is not a newline. + The \N{U+hhh..} escape sequence is recognized only when PCRE2 is oper- + ating in UTF mode. Perl also uses \N{name} to specify characters by + Unicode name; PCRE2 does not support this. Note that when \N is not + followed by an opening brace (curly bracket) it has an entirely differ- + ent meaning, matching any character that is not a newline. - There are some legacy applications where the escape sequence \r is - expected to match a newline. If the PCRE2_EXTRA_ESCAPED_CR_IS_LF option - is set, \r in a pattern is converted to \n so that it matches a LF + There are some legacy applications where the escape sequence \r is ex- + pected to match a newline. If the PCRE2_EXTRA_ESCAPED_CR_IS_LF option + is set, \r in a pattern is converted to \n so that it matches a LF (linefeed) instead of a CR (carriage return) character. - The precise effect of \cx on ASCII characters is as follows: if x is a - lower case letter, it is converted to upper case. Then bit 6 of the + The precise effect of \cx on ASCII characters is as follows: if x is a + lower case letter, it is converted to upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A - (A is 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes - hex 7B (; is 3B). If the code unit following \c has a value less than + (A is 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes + hex 7B (; is 3B). If the code unit following \c has a value less than 32 or greater than 126, a compile-time error occurs. - When PCRE2 is compiled in EBCDIC mode, \N{U+hhh..} is not supported. + When PCRE2 is compiled in EBCDIC mode, \N{U+hhh..} is not supported. \a, \e, \f, \n, \r, and \t generate the appropriate EBCDIC code values. The \c escape is processed as specified for Perl in the perlebcdic doc- - ument. The only characters that are allowed after \c are A-Z, a-z, or - one of @, [, \, ], ^, _, or ?. Any other character provokes a compile- - time error. The sequence \c@ encodes character code 0; after \c the - letters (in either case) encode characters 1-26 (hex 01 to hex 1A); [, - \, ], ^, and _ encode characters 27-31 (hex 1B to hex 1F), and \c? - becomes either 255 (hex FF) or 95 (hex 5F). - - Thus, apart from \c?, these escapes generate the same character code - values as they do in an ASCII environment, though the meanings of the - values mostly differ. For example, \cG always generates code value 7, + ument. The only characters that are allowed after \c are A-Z, a-z, or + one of @, [, \, ], ^, _, or ?. Any other character provokes a compile- + time error. The sequence \c@ encodes character code 0; after \c the + letters (in either case) encode characters 1-26 (hex 01 to hex 1A); [, + \, ], ^, and _ encode characters 27-31 (hex 1B to hex 1F), and \c? be- + comes either 255 (hex FF) or 95 (hex 5F). + + Thus, apart from \c?, these escapes generate the same character code + values as they do in an ASCII environment, though the meanings of the + values mostly differ. For example, \cG always generates code value 7, which is BEL in ASCII but DEL in EBCDIC. - The sequence \c? generates DEL (127, hex 7F) in an ASCII environment, - but because 127 is not a control character in EBCDIC, Perl makes it - generate the APC character. Unfortunately, there are several variants - of EBCDIC. In most of them the APC character has the value 255 (hex - FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If + The sequence \c? generates DEL (127, hex 7F) in an ASCII environment, + but because 127 is not a control character in EBCDIC, Perl makes it + generate the APC character. Unfortunately, there are several variants + of EBCDIC. In most of them the APC character has the value 255 (hex + FF), but in the one Perl calls POSIX-BC its value is 95 (hex 5F). If certain other characters have POSIX-BC values, PCRE2 makes \c? generate 95; otherwise it generates 255. - After \0 up to two further octal digits are read. If there are fewer - than two digits, just those that are present are used. Thus the - sequence \0\x\015 specifies two binary zeros followed by a CR character + After \0 up to two further octal digits are read. If there are fewer + than two digits, just those that are present are used. Thus the se- + quence \0\x\015 specifies two binary zeros followed by a CR character (code value 13). Make sure you supply two digits after the initial zero if the pattern character that follows is itself an octal digit. - The escape \o must be followed by a sequence of octal digits, enclosed - in braces. An error occurs if this is not the case. This escape is a - recent addition to Perl; it provides way of specifying character code - points as octal numbers greater than 0777, and it also allows octal + The escape \o must be followed by a sequence of octal digits, enclosed + in braces. An error occurs if this is not the case. This escape is a + recent addition to Perl; it provides way of specifying character code + points as octal numbers greater than 0777, and it also allows octal numbers and backreferences to be unambiguously specified. For greater clarity and unambiguity, it is best to avoid following \ by @@ -6470,16 +6602,16 @@ BACKSLASH Outside a character class, PCRE2 reads the digit and any following dig- its as a decimal number. If the number is less than 10, begins with the - digit 8 or 9, or if there are at least that many previous capture - groups in the expression, the entire sequence is taken as a backrefer- - ence. A description of how this works is given later, following the - discussion of parenthesized groups. Otherwise, up to three octal dig- + digit 8 or 9, or if there are at least that many previous capture + groups in the expression, the entire sequence is taken as a backrefer- + ence. A description of how this works is given later, following the + discussion of parenthesized groups. Otherwise, up to three octal dig- its are read to form a character code. - Inside a character class, PCRE2 handles \8 and \9 as the literal char- - acters "8" and "9", and otherwise reads up to three octal digits fol- + Inside a character class, PCRE2 handles \8 and \9 as the literal char- + acters "8" and "9", and otherwise reads up to three octal digits fol- lowing the backslash, using them to generate a data character. Any sub- - sequent digits stand for themselves. For example, outside a character + sequent digits stand for themselves. For example, outside a character class: \040 is another way of writing an ASCII space @@ -6496,13 +6628,13 @@ BACKSLASH the value 255 (decimal) \81 is always a backreference - Note that octal values of 100 or greater that are specified using this - syntax must not be introduced by a leading zero, because no more than + Note that octal values of 100 or greater that are specified using this + syntax must not be introduced by a leading zero, because no more than three octal digits are ever read. Constraints on character values - Characters that are specified using octal or hexadecimal numbers are + Characters that are specified using octal or hexadecimal numbers are limited to certain values, as follows: 8-bit non-UTF mode no greater than 0xff @@ -6511,45 +6643,45 @@ BACKSLASH All UTF modes no greater than 0x10ffff and a valid code point Invalid Unicode code points are all those in the range 0xd800 to 0xdfff - (the so-called "surrogate" code points). The check for these can be - disabled by the caller of pcre2_compile() by setting the option - PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES. However, this is possible only in - UTF-8 and UTF-32 modes, because these values are not representable in + (the so-called "surrogate" code points). The check for these can be + disabled by the caller of pcre2_compile() by setting the option + PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES. However, this is possible only in + UTF-8 and UTF-32 modes, because these values are not representable in UTF-16. Escape sequences in character classes All the sequences that define a single character value can be used both - inside and outside character classes. In addition, inside a character + inside and outside character classes. In addition, inside a character class, \b is interpreted as the backspace character (hex 08). When not followed by an opening brace, \N is not allowed in a character - class. \B, \R, and \X are not special inside a character class. Like - other unrecognized alphabetic escape sequences, they cause an error. + class. \B, \R, and \X are not special inside a character class. Like + other unrecognized alphabetic escape sequences, they cause an error. Outside a character class, these sequences have different meanings. Unsupported escape sequences - In Perl, the sequences \F, \l, \L, \u, and \U are recognized by its - string handler and used to modify the case of following characters. By - default, PCRE2 does not support these escape sequences in patterns. - However, if either of the PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX - options is set, \U matches a "U" character, and \u can be used to - define a character by code point, as described above. + In Perl, the sequences \F, \l, \L, \u, and \U are recognized by its + string handler and used to modify the case of following characters. By + default, PCRE2 does not support these escape sequences in patterns. + However, if either of the PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX op- + tions is set, \U matches a "U" character, and \u can be used to define + a character by code point, as described above. Absolute and relative backreferences - The sequence \g followed by a signed or unsigned number, optionally - enclosed in braces, is an absolute or relative backreference. A named - backreference can be coded as \g{name}. Backreferences are discussed + The sequence \g followed by a signed or unsigned number, optionally en- + closed in braces, is an absolute or relative backreference. A named + backreference can be coded as \g{name}. Backreferences are discussed later, following the discussion of parenthesized groups. Absolute and relative subroutine calls - For compatibility with Oniguruma, the non-Perl syntax \g followed by a + For compatibility with Oniguruma, the non-Perl syntax \g followed by a name or a number enclosed either in angle brackets or single quotes, is - an alternative syntax for referencing a capture group as a subroutine. - Details are discussed later. Note that \g{...} (Perl syntax) and + an alternative syntax for referencing a capture group as a subroutine. + Details are discussed later. Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are not synonymous. The former is a backref- erence; the latter is a subroutine call. @@ -6569,58 +6701,58 @@ BACKSLASH \w any "word" character \W any "non-word" character - The \N escape sequence has the same meaning as the "." metacharacter - when PCRE2_DOTALL is not set, but setting PCRE2_DOTALL does not change + The \N escape sequence has the same meaning as the "." metacharacter + when PCRE2_DOTALL is not set, but setting PCRE2_DOTALL does not change the meaning of \N. Note that when \N is followed by an opening brace it has a different meaning. See the section entitled "Non-printing charac- - ters" above for details. Perl also uses \N{name} to specify characters + ters" above for details. Perl also uses \N{name} to specify characters by Unicode name; PCRE2 does not support this. - Each pair of lower and upper case escape sequences partitions the com- - plete set of characters into two disjoint sets. Any given character - matches one, and only one, of each pair. The sequences can appear both - inside and outside character classes. They each match one character of - the appropriate type. If the current matching point is at the end of - the subject string, all of them fail, because there is no character to + Each pair of lower and upper case escape sequences partitions the com- + plete set of characters into two disjoint sets. Any given character + matches one, and only one, of each pair. The sequences can appear both + inside and outside character classes. They each match one character of + the appropriate type. If the current matching point is at the end of + the subject string, all of them fail, because there is no character to match. - The default \s characters are HT (9), LF (10), VT (11), FF (12), CR - (13), and space (32), which are defined as white space in the "C" - locale. This list may vary if locale-specific matching is taking place. - For example, in some locales the "non-breaking space" character (\xA0) + The default \s characters are HT (9), LF (10), VT (11), FF (12), CR + (13), and space (32), which are defined as white space in the "C" lo- + cale. This list may vary if locale-specific matching is taking place. + For example, in some locales the "non-breaking space" character (\xA0) is recognized as white space, and in others the VT character is not. - A "word" character is an underscore or any character that is a letter - or digit. By default, the definition of letters and digits is con- + A "word" character is an underscore or any character that is a letter + or digit. By default, the definition of letters and digits is con- trolled by PCRE2's low-valued character tables, and may vary if locale- specific matching is taking place (see "Locale support" in the pcre2api - page). For example, in a French locale such as "fr_FR" in Unix-like - systems, or "french" in Windows, some character codes greater than 127 - are used for accented letters, and these are then matched by \w. The + page). For example, in a French locale such as "fr_FR" in Unix-like + systems, or "french" in Windows, some character codes greater than 127 + are used for accented letters, and these are then matched by \w. The use of locales with Unicode is discouraged. - By default, characters whose code points are greater than 127 never + By default, characters whose code points are greater than 127 never match \d, \s, or \w, and always match \D, \S, and \W, although this may - be different for characters in the range 128-255 when locale-specific - matching is happening. These escape sequences retain their original - meanings from before Unicode support was available, mainly for effi- - ciency reasons. If the PCRE2_UCP option is set, the behaviour is - changed so that Unicode properties are used to determine character + be different for characters in the range 128-255 when locale-specific + matching is happening. These escape sequences retain their original + meanings from before Unicode support was available, mainly for effi- + ciency reasons. If the PCRE2_UCP option is set, the behaviour is + changed so that Unicode properties are used to determine character types, as follows: \d any character that matches \p{Nd} (decimal digit) \s any character that matches \p{Z} or \h or \v \w any character that matches \p{L} or \p{N}, plus underscore - The upper case escapes match the inverse sets of characters. Note that - \d matches only decimal digits, whereas \w matches any Unicode digit, + The upper case escapes match the inverse sets of characters. Note that + \d matches only decimal digits, whereas \w matches any Unicode digit, as well as any Unicode letter, and underscore. Note also that PCRE2_UCP - affects \b, and \B because they are defined in terms of \w and \W. + affects \b, and \B because they are defined in terms of \w and \W. Matching these sequences is noticeably slower when PCRE2_UCP is set. - The sequences \h, \H, \v, and \V, in contrast to the other sequences, - which match only ASCII characters by default, always match a specific - list of code points, whether or not PCRE2_UCP is set. The horizontal + The sequences \h, \H, \v, and \V, in contrast to the other sequences, + which match only ASCII characters by default, always match a specific + list of code points, whether or not PCRE2_UCP is set. The horizontal space characters are: U+0009 Horizontal tab (HT) @@ -6653,36 +6785,36 @@ BACKSLASH U+2028 Line separator U+2029 Paragraph separator - In 8-bit, non-UTF-8 mode, only the characters with code points less + In 8-bit, non-UTF-8 mode, only the characters with code points less than 256 are relevant. Newline sequences - Outside a character class, by default, the escape sequence \R matches - any Unicode newline sequence. In 8-bit non-UTF-8 mode \R is equivalent + Outside a character class, by default, the escape sequence \R matches + any Unicode newline sequence. In 8-bit non-UTF-8 mode \R is equivalent to the following: (?>\r\n|\n|\x0b|\f|\r|\x85) - This is an example of an "atomic group", details of which are given - below. This particular group matches either the two-character sequence - CR followed by LF, or one of the single characters LF (linefeed, - U+000A), VT (vertical tab, U+000B), FF (form feed, U+000C), CR (car- - riage return, U+000D), or NEL (next line, U+0085). Because this is an - atomic group, the two-character sequence is treated as a single unit + This is an example of an "atomic group", details of which are given be- + low. This particular group matches either the two-character sequence + CR followed by LF, or one of the single characters LF (linefeed, + U+000A), VT (vertical tab, U+000B), FF (form feed, U+000C), CR (car- + riage return, U+000D), or NEL (next line, U+0085). Because this is an + atomic group, the two-character sequence is treated as a single unit that cannot be split. In other modes, two additional characters whose code points are greater than 255 are added: LS (line separator, U+2028) and PS (paragraph sepa- - rator, U+2029). Unicode support is not needed for these characters to + rator, U+2029). Unicode support is not needed for these characters to be recognized. It is possible to restrict \R to match only CR, LF, or CRLF (instead of - the complete set of Unicode line endings) by setting the option - PCRE2_BSR_ANYCRLF at compile time. (BSR is an abbrevation for "back- + the complete set of Unicode line endings) by setting the option + PCRE2_BSR_ANYCRLF at compile time. (BSR is an abbrevation for "back- slash R".) This can be made the default when PCRE2 is built; if this is - the case, the other behaviour can be requested via the PCRE2_BSR_UNI- - CODE option. It is also possible to specify these settings by starting + the case, the other behaviour can be requested via the PCRE2_BSR_UNI- + CODE option. It is also possible to specify these settings by starting a pattern string with one of the following sequences: (*BSR_ANYCRLF) CR, LF, or CRLF only @@ -6690,27 +6822,27 @@ BACKSLASH These override the default and the options given to the compiling func- tion. Note that these special settings, which are not Perl-compatible, - are recognized only at the very start of a pattern, and that they must - be in upper case. If more than one of them is present, the last one is - used. They can be combined with a change of newline convention; for - example, a pattern can start with: + are recognized only at the very start of a pattern, and that they must + be in upper case. If more than one of them is present, the last one is + used. They can be combined with a change of newline convention; for ex- + ample, a pattern can start with: (*ANY)(*BSR_ANYCRLF) - They can also be combined with the (*UTF) or (*UCP) special sequences. - Inside a character class, \R is treated as an unrecognized escape - sequence, and causes an error. + They can also be combined with the (*UTF) or (*UCP) special sequences. + Inside a character class, \R is treated as an unrecognized escape se- + quence, and causes an error. Unicode character properties - When PCRE2 is built with Unicode support (the default), three addi- - tional escape sequences that match characters with specific properties + When PCRE2 is built with Unicode support (the default), three addi- + tional escape sequences that match characters with specific properties are available. They can be used in any mode, though in 8-bit and 16-bit - non-UTF modes these sequences are of course limited to testing charac- - ters whose code points are less than U+0100 and U+10000, respectively. - In 32-bit non-UTF mode, code points greater than 0x10ffff (the Unicode - limit) may be encountered. These are all treated as being in the - Unknown script and with an unassigned type. The extra escape sequences + non-UTF modes these sequences are of course limited to testing charac- + ters whose code points are less than U+0100 and U+10000, respectively. + In 32-bit non-UTF mode, code points greater than 0x10ffff (the Unicode + limit) may be encountered. These are all treated as being in the Un- + known script and with an unassigned type. The extra escape sequences are: \p{xx} a character with the xx property @@ -6718,15 +6850,15 @@ BACKSLASH \X a Unicode extended grapheme cluster The property names represented by xx above are case-sensitive. There is - support for Unicode script names, Unicode general category properties, - "Any", which matches any character (including newline), and some spe- - cial PCRE2 properties (described in the next section). Other Perl + support for Unicode script names, Unicode general category properties, + "Any", which matches any character (including newline), and some spe- + cial PCRE2 properties (described in the next section). Other Perl properties such as "InMusicalSymbols" are not supported by PCRE2. Note - that \P{Any} does not match any characters, so always causes a match + that \P{Any} does not match any characters, so always causes a match failure. Sets of Unicode characters are defined as belonging to certain scripts. - A character from one of these sets can be matched using a script name. + A character from one of these sets can be matched using a script name. For example: \p{Greek} @@ -6734,33 +6866,34 @@ BACKSLASH Unassigned characters (and in non-UTF 32-bit mode, characters with code points greater than 0x10FFFF) are assigned the "Unknown" script. Others - that are not part of an identified script are lumped together as "Com- + that are not part of an identified script are lumped together as "Com- mon". The current list of scripts is: - Adlam, Ahom, Anatolian_Hieroglyphs, Arabic, Armenian, Avestan, Bali- - nese, Bamum, Bassa_Vah, Batak, Bengali, Bhaiksuki, Bopomofo, Brahmi, - Braille, Buginese, Buhid, Canadian_Aboriginal, Carian, Caucasian_Alba- - nian, Chakma, Cham, Cherokee, Common, Coptic, Cuneiform, Cypriot, - Cyrillic, Deseret, Devanagari, Dogra, Duployan, Egyptian_Hieroglyphs, - Elbasan, Ethiopic, Georgian, Glagolitic, Gothic, Grantha, Greek, - Gujarati, Gunjala_Gondi, Gurmukhi, Han, Hangul, Hanifi_Rohingya, - Hanunoo, Hatran, Hebrew, Hiragana, Imperial_Aramaic, Inherited, - Inscriptional_Pahlavi, Inscriptional_Parthian, Javanese, Kaithi, Kan- - nada, Katakana, Kayah_Li, Kharoshthi, Khmer, Khojki, Khudawadi, Lao, - Latin, Lepcha, Limbu, Linear_A, Linear_B, Lisu, Lycian, Lydian, Maha- - jani, Makasar, Malayalam, Mandaic, Manichaean, Marchen, Masaram_Gondi, - Medefaidrin, Meetei_Mayek, Mende_Kikakui, Meroitic_Cursive, - Meroitic_Hieroglyphs, Miao, Modi, Mongolian, Mro, Multani, Myanmar, - Nabataean, New_Tai_Lue, Newa, Nko, Nushu, Ogham, Ol_Chiki, Old_Hungar- + Adlam, Ahom, Anatolian_Hieroglyphs, Arabic, Armenian, Avestan, Bali- + nese, Bamum, Bassa_Vah, Batak, Bengali, Bhaiksuki, Bopomofo, Brahmi, + Braille, Buginese, Buhid, Canadian_Aboriginal, Carian, Caucasian_Alba- + nian, Chakma, Cham, Cherokee, Chorasmian, Common, Coptic, Cuneiform, + Cypriot, Cyrillic, Deseret, Devanagari, Dives_Akuru, Dogra, Duployan, + Egyptian_Hieroglyphs, Elbasan, Elymaic, Ethiopic, Georgian, Glagolitic, + Gothic, Grantha, Greek, Gujarati, Gunjala_Gondi, Gurmukhi, Han, Hangul, + Hanifi_Rohingya, Hanunoo, Hatran, Hebrew, Hiragana, Imperial_Aramaic, + Inherited, Inscriptional_Pahlavi, Inscriptional_Parthian, Javanese, + Kaithi, Kannada, Katakana, Kayah_Li, Kharoshthi, Khitan_Small_Script, + Khmer, Khojki, Khudawadi, Lao, Latin, Lepcha, Limbu, Linear_A, Lin- + ear_B, Lisu, Lycian, Lydian, Mahajani, Makasar, Malayalam, Mandaic, + Manichaean, Marchen, Masaram_Gondi, Medefaidrin, Meetei_Mayek, + Mende_Kikakui, Meroitic_Cursive, Meroitic_Hieroglyphs, Miao, Modi, Mon- + golian, Mro, Multani, Myanmar, Nabataean, Nandinagari, New_Tai_Lue, + Newa, Nko, Nushu, Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungar- ian, Old_Italic, Old_North_Arabian, Old_Permic, Old_Persian, Old_Sog- - dian, Old_South_Arabian, Old_Turkic, Oriya, Osage, Osmanya, - Pahawh_Hmong, Palmyrene, Pau_Cin_Hau, Phags_Pa, Phoenician, + dian, Old_South_Arabian, Old_Turkic, Oriya, Osage, Osmanya, Pa- + hawh_Hmong, Palmyrene, Pau_Cin_Hau, Phags_Pa, Phoenician, Psalter_Pahlavi, Rejang, Runic, Samaritan, Saurashtra, Sharada, Sha- vian, Siddham, SignWriting, Sinhala, Sogdian, Sora_Sompeng, Soyombo, Sundanese, Syloti_Nagri, Syriac, Tagalog, Tagbanwa, Tai_Le, Tai_Tham, Tai_Viet, Takri, Tamil, Tangut, Telugu, Thaana, Thai, Tibetan, Tifi- - nagh, Tirhuta, Ugaritic, Unknown, Vai, Warang_Citi, Yi, Zan- - abazar_Square. + nagh, Tirhuta, Ugaritic, Unknown, Vai, Wancho, Warang_Citi, Yezidi, Yi, + Zanabazar_Square. Each character has exactly one Unicode general category property, spec- ified by a two-letter abbreviation. For compatibility with Perl, nega- @@ -6882,8 +7015,8 @@ BACKSLASH only by a T character. 4. Do not end before extending characters or spacing marks or the - "zero-width joiner" character. Characters with the "mark" property - always have the "extend" grapheme breaking property. + "zero-width joiner" character. Characters with the "mark" property al- + ways have the "extend" grapheme breaking property. 5. Do not end after prepend characters. @@ -6892,17 +7025,17 @@ BACKSLASH property. Extend and ZWJ characters are allowed between the charac- ters. - 7. Do not break within emoji flag sequences. That is, do not break - between regional indicator (RI) characters if there are an odd number - of RI characters before the break point. + 7. Do not break within emoji flag sequences. That is, do not break be- + tween regional indicator (RI) characters if there are an odd number of + RI characters before the break point. 8. Otherwise, end the cluster. PCRE2's additional properties As well as the standard Unicode properties described above, PCRE2 sup- - ports four more that make it possible to convert traditional escape - sequences such as \w and \s to use Unicode properties. PCRE2 uses these + ports four more that make it possible to convert traditional escape se- + quences such as \w and \s to use Unicode properties. PCRE2 uses these non-standard, non-Perl properties internally when PCRE2_UCP is set. However, they may also be used explicitly. These properties are: @@ -6914,8 +7047,8 @@ BACKSLASH Xan matches characters that have either the L (letter) or the N (num- ber) property. Xps matches the characters tab, linefeed, vertical tab, form feed, or carriage return, and any other character that has the Z - (separator) property. Xsp is the same as Xps; in PCRE1 it used to - exclude vertical tab, for Perl compatibility, but Perl changed. Xwd + (separator) property. Xsp is the same as Xps; in PCRE1 it used to ex- + clude vertical tab, for Perl compatibility, but Perl changed. Xwd matches the same characters as Xan, plus underscore. There is another non-standard property, Xuc, which matches any charac- @@ -6931,8 +7064,8 @@ BACKSLASH Resetting the match start In normal use, the escape sequence \K causes any previously matched - characters not to be included in the final matched sequence that is - returned. For example, the pattern: + characters not to be included in the final matched sequence that is re- + turned. For example, the pattern: foo\Kbar @@ -6953,27 +7086,28 @@ BACKSLASH matches "foobar", the first substring is still set to "foo". - Perl documents that the use of \K within assertions is "not well - defined". In PCRE2, \K is acted upon when it occurs inside positive - assertions, but is ignored in negative assertions. Note that when a - pattern such as (?=ab\K) matches, the reported start of the match can - be greater than the end of the match. Using \K in a lookbehind asser- - tion at the start of a pattern can also lead to odd effects. For exam- - ple, consider this pattern: + Perl used to document that the use of \K within lookaround assertions + is "not well defined", but from version 5.32.0 Perl does not support + this usage at all. In PCRE2, \K is acted upon when it occurs inside + positive assertions, but is ignored in negative assertions. Note that + when a pattern such as (?=ab\K) matches, the reported start of the + match can be greater than the end of the match. Using \K in a lookbe- + hind assertion at the start of a pattern can also lead to odd effects. + For example, consider this pattern: (?<=\Kfoo)bar - If the subject is "foobar", a call to pcre2_match() with a starting - offset of 3 succeeds and reports the matching string as "foobar", that - is, the start of the reported match is earlier than where the match + If the subject is "foobar", a call to pcre2_match() with a starting + offset of 3 succeeds and reports the matching string as "foobar", that + is, the start of the reported match is earlier than where the match started. Simple assertions - The final use of backslash is for certain simple assertions. An asser- - tion specifies a condition that has to be met at a particular point in - a match, without consuming any characters from the subject string. The - use of groups for more complicated assertions is described below. The + The final use of backslash is for certain simple assertions. An asser- + tion specifies a condition that has to be met at a particular point in + a match, without consuming any characters from the subject string. The + use of groups for more complicated assertions is described below. The backslashed assertions are: \b matches at a word boundary @@ -6984,191 +7118,191 @@ BACKSLASH \z matches only at the end of the subject \G matches at the first matching position in the subject - Inside a character class, \b has a different meaning; it matches the - backspace character. If any other of these assertions appears in a + Inside a character class, \b has a different meaning; it matches the + backspace character. If any other of these assertions appears in a character class, an "invalid escape sequence" error is generated. - A word boundary is a position in the subject string where the current - character and the previous character do not both match \w or \W (i.e. - one matches \w and the other matches \W), or the start or end of the - string if the first or last character matches \w, respectively. When - PCRE2 is built with Unicode support, the meanings of \w and \W can be - changed by setting the PCRE2_UCP option. When this is done, it also - affects \b and \B. Neither PCRE2 nor Perl has a separate "start of - word" or "end of word" metasequence. However, whatever follows \b nor- - mally determines which it is. For example, the fragment \ba matches "a" - at the start of a word. - - The \A, \Z, and \z assertions differ from the traditional circumflex + A word boundary is a position in the subject string where the current + character and the previous character do not both match \w or \W (i.e. + one matches \w and the other matches \W), or the start or end of the + string if the first or last character matches \w, respectively. When + PCRE2 is built with Unicode support, the meanings of \w and \W can be + changed by setting the PCRE2_UCP option. When this is done, it also af- + fects \b and \B. Neither PCRE2 nor Perl has a separate "start of word" + or "end of word" metasequence. However, whatever follows \b normally + determines which it is. For example, the fragment \ba matches "a" at + the start of a word. + + The \A, \Z, and \z assertions differ from the traditional circumflex and dollar (described in the next section) in that they only ever match - at the very start and end of the subject string, whatever options are - set. Thus, they are independent of multiline mode. These three asser- - tions are not affected by the PCRE2_NOTBOL or PCRE2_NOTEOL options, - which affect only the behaviour of the circumflex and dollar metachar- - acters. However, if the startoffset argument of pcre2_match() is non- - zero, indicating that matching is to start at a point other than the - beginning of the subject, \A can never match. The difference between - \Z and \z is that \Z matches before a newline at the end of the string + at the very start and end of the subject string, whatever options are + set. Thus, they are independent of multiline mode. These three asser- + tions are not affected by the PCRE2_NOTBOL or PCRE2_NOTEOL options, + which affect only the behaviour of the circumflex and dollar metachar- + acters. However, if the startoffset argument of pcre2_match() is non- + zero, indicating that matching is to start at a point other than the + beginning of the subject, \A can never match. The difference between + \Z and \z is that \Z matches before a newline at the end of the string as well as at the very end, whereas \z matches only at the end. - The \G assertion is true only when the current matching position is at - the start point of the matching process, as specified by the startoff- - set argument of pcre2_match(). It differs from \A when the value of - startoffset is non-zero. By calling pcre2_match() multiple times with - appropriate arguments, you can mimic Perl's /g option, and it is in + The \G assertion is true only when the current matching position is at + the start point of the matching process, as specified by the startoff- + set argument of pcre2_match(). It differs from \A when the value of + startoffset is non-zero. By calling pcre2_match() multiple times with + appropriate arguments, you can mimic Perl's /g option, and it is in this kind of implementation where \G can be useful. - Note, however, that PCRE2's implementation of \G, being true at the - starting character of the matching process, is subtly different from - Perl's, which defines it as true at the end of the previous match. In - Perl, these can be different when the previously matched string was + Note, however, that PCRE2's implementation of \G, being true at the + starting character of the matching process, is subtly different from + Perl's, which defines it as true at the end of the previous match. In + Perl, these can be different when the previously matched string was empty. Because PCRE2 does just one match at a time, it cannot reproduce this behaviour. - If all the alternatives of a pattern begin with \G, the expression is + If all the alternatives of a pattern begin with \G, the expression is anchored to the starting match position, and the "anchored" flag is set in the compiled regular expression. CIRCUMFLEX AND DOLLAR - The circumflex and dollar metacharacters are zero-width assertions. - That is, they test for a particular condition being true without con- + The circumflex and dollar metacharacters are zero-width assertions. + That is, they test for a particular condition being true without con- suming any characters from the subject string. These two metacharacters - are concerned with matching the starts and ends of lines. If the new- - line convention is set so that only the two-character sequence CRLF is - recognized as a newline, isolated CR and LF characters are treated as + are concerned with matching the starts and ends of lines. If the new- + line convention is set so that only the two-character sequence CRLF is + recognized as a newline, isolated CR and LF characters are treated as ordinary data characters, and are not recognized as newlines. Outside a character class, in the default matching mode, the circumflex - character is an assertion that is true only if the current matching - point is at the start of the subject string. If the startoffset argu- - ment of pcre2_match() is non-zero, or if PCRE2_NOTBOL is set, circum- - flex can never match if the PCRE2_MULTILINE option is unset. Inside a - character class, circumflex has an entirely different meaning (see - below). - - Circumflex need not be the first character of the pattern if a number - of alternatives are involved, but it should be the first thing in each - alternative in which it appears if the pattern is ever to match that - branch. If all possible alternatives start with a circumflex, that is, - if the pattern is constrained to match only at the start of the sub- - ject, it is said to be an "anchored" pattern. (There are also other + character is an assertion that is true only if the current matching + point is at the start of the subject string. If the startoffset argu- + ment of pcre2_match() is non-zero, or if PCRE2_NOTBOL is set, circum- + flex can never match if the PCRE2_MULTILINE option is unset. Inside a + character class, circumflex has an entirely different meaning (see be- + low). + + Circumflex need not be the first character of the pattern if a number + of alternatives are involved, but it should be the first thing in each + alternative in which it appears if the pattern is ever to match that + branch. If all possible alternatives start with a circumflex, that is, + if the pattern is constrained to match only at the start of the sub- + ject, it is said to be an "anchored" pattern. (There are also other constructs that can cause a pattern to be anchored.) - The dollar character is an assertion that is true only if the current - matching point is at the end of the subject string, or immediately - before a newline at the end of the string (by default), unless - PCRE2_NOTEOL is set. Note, however, that it does not actually match the - newline. Dollar need not be the last character of the pattern if a num- - ber of alternatives are involved, but it should be the last item in any - branch in which it appears. Dollar has no special meaning in a charac- + The dollar character is an assertion that is true only if the current + matching point is at the end of the subject string, or immediately be- + fore a newline at the end of the string (by default), unless PCRE2_NO- + TEOL is set. Note, however, that it does not actually match the new- + line. Dollar need not be the last character of the pattern if a number + of alternatives are involved, but it should be the last item in any + branch in which it appears. Dollar has no special meaning in a charac- ter class. - The meaning of dollar can be changed so that it matches only at the - very end of the string, by setting the PCRE2_DOLLAR_ENDONLY option at + The meaning of dollar can be changed so that it matches only at the + very end of the string, by setting the PCRE2_DOLLAR_ENDONLY option at compile time. This does not affect the \Z assertion. The meanings of the circumflex and dollar metacharacters are changed if - the PCRE2_MULTILINE option is set. When this is the case, a dollar - character matches before any newlines in the string, as well as at the - very end, and a circumflex matches immediately after internal newlines - as well as at the start of the subject string. It does not match after - a newline that ends the string, for compatibility with Perl. However, + the PCRE2_MULTILINE option is set. When this is the case, a dollar + character matches before any newlines in the string, as well as at the + very end, and a circumflex matches immediately after internal newlines + as well as at the start of the subject string. It does not match after + a newline that ends the string, for compatibility with Perl. However, this can be changed by setting the PCRE2_ALT_CIRCUMFLEX option. - For example, the pattern /^abc$/ matches the subject string "def\nabc" - (where \n represents a newline) in multiline mode, but not otherwise. - Consequently, patterns that are anchored in single line mode because - all branches start with ^ are not anchored in multiline mode, and a - match for circumflex is possible when the startoffset argument of - pcre2_match() is non-zero. The PCRE2_DOLLAR_ENDONLY option is ignored + For example, the pattern /^abc$/ matches the subject string "def\nabc" + (where \n represents a newline) in multiline mode, but not otherwise. + Consequently, patterns that are anchored in single line mode because + all branches start with ^ are not anchored in multiline mode, and a + match for circumflex is possible when the startoffset argument of + pcre2_match() is non-zero. The PCRE2_DOLLAR_ENDONLY option is ignored if PCRE2_MULTILINE is set. - When the newline convention (see "Newline conventions" below) recog- - nizes the two-character sequence CRLF as a newline, this is preferred, - even if the single characters CR and LF are also recognized as new- - lines. For example, if the newline convention is "any", a multiline - mode circumflex matches before "xyz" in the string "abc\r\nxyz" rather - than after CR, even though CR on its own is a valid newline. (It also + When the newline convention (see "Newline conventions" below) recog- + nizes the two-character sequence CRLF as a newline, this is preferred, + even if the single characters CR and LF are also recognized as new- + lines. For example, if the newline convention is "any", a multiline + mode circumflex matches before "xyz" in the string "abc\r\nxyz" rather + than after CR, even though CR on its own is a valid newline. (It also matches at the very start of the string, of course.) - Note that the sequences \A, \Z, and \z can be used to match the start - and end of the subject in both modes, and if all branches of a pattern - start with \A it is always anchored, whether or not PCRE2_MULTILINE is + Note that the sequences \A, \Z, and \z can be used to match the start + and end of the subject in both modes, and if all branches of a pattern + start with \A it is always anchored, whether or not PCRE2_MULTILINE is set. FULL STOP (PERIOD, DOT) AND \N Outside a character class, a dot in the pattern matches any one charac- - ter in the subject string except (by default) a character that signi- + ter in the subject string except (by default) a character that signi- fies the end of a line. - When a line ending is defined as a single character, dot never matches - that character; when the two-character sequence CRLF is used, dot does - not match CR if it is immediately followed by LF, but otherwise it - matches all characters (including isolated CRs and LFs). When any Uni- - code line endings are being recognized, dot does not match CR or LF or + When a line ending is defined as a single character, dot never matches + that character; when the two-character sequence CRLF is used, dot does + not match CR if it is immediately followed by LF, but otherwise it + matches all characters (including isolated CRs and LFs). When any Uni- + code line endings are being recognized, dot does not match CR or LF or any of the other line ending characters. - The behaviour of dot with regard to newlines can be changed. If the - PCRE2_DOTALL option is set, a dot matches any one character, without - exception. If the two-character sequence CRLF is present in the sub- + The behaviour of dot with regard to newlines can be changed. If the + PCRE2_DOTALL option is set, a dot matches any one character, without + exception. If the two-character sequence CRLF is present in the sub- ject string, it takes two dots to match it. - The handling of dot is entirely independent of the handling of circum- - flex and dollar, the only relationship being that they both involve + The handling of dot is entirely independent of the handling of circum- + flex and dollar, the only relationship being that they both involve newlines. Dot has no special meaning in a character class. - The escape sequence \N when not followed by an opening brace behaves - like a dot, except that it is not affected by the PCRE2_DOTALL option. - In other words, it matches any character except one that signifies the + The escape sequence \N when not followed by an opening brace behaves + like a dot, except that it is not affected by the PCRE2_DOTALL option. + In other words, it matches any character except one that signifies the end of a line. When \N is followed by an opening brace it has a different meaning. See - the section entitled "Non-printing characters" above for details. Perl - also uses \N{name} to specify characters by Unicode name; PCRE2 does + the section entitled "Non-printing characters" above for details. Perl + also uses \N{name} to specify characters by Unicode name; PCRE2 does not support this. MATCHING A SINGLE CODE UNIT - Outside a character class, the escape sequence \C matches any one code - unit, whether or not a UTF mode is set. In the 8-bit library, one code - unit is one byte; in the 16-bit library it is a 16-bit unit; in the - 32-bit library it is a 32-bit unit. Unlike a dot, \C always matches - line-ending characters. The feature is provided in Perl in order to + Outside a character class, the escape sequence \C matches any one code + unit, whether or not a UTF mode is set. In the 8-bit library, one code + unit is one byte; in the 16-bit library it is a 16-bit unit; in the + 32-bit library it is a 32-bit unit. Unlike a dot, \C always matches + line-ending characters. The feature is provided in Perl in order to match individual bytes in UTF-8 mode, but it is unclear how it can use- fully be used. - Because \C breaks up characters into individual code units, matching - one unit with \C in UTF-8 or UTF-16 mode means that the rest of the - string may start with a malformed UTF character. This has undefined - results, because PCRE2 assumes that it is matching character by charac- - ter in a valid UTF string (by default it checks the subject string's - validity at the start of processing unless the PCRE2_NO_UTF_CHECK - option is used). + Because \C breaks up characters into individual code units, matching + one unit with \C in UTF-8 or UTF-16 mode means that the rest of the + string may start with a malformed UTF character. This has undefined re- + sults, because PCRE2 assumes that it is matching character by character + in a valid UTF string (by default it checks the subject string's valid- + ity at the start of processing unless the PCRE2_NO_UTF_CHECK or + PCRE2_MATCH_INVALID_UTF option is used). - An application can lock out the use of \C by setting the - PCRE2_NEVER_BACKSLASH_C option when compiling a pattern. It is also + An application can lock out the use of \C by setting the + PCRE2_NEVER_BACKSLASH_C option when compiling a pattern. It is also possible to build PCRE2 with the use of \C permanently disabled. - PCRE2 does not allow \C to appear in lookbehind assertions (described - below) in UTF-8 or UTF-16 modes, because this would make it impossible - to calculate the length of the lookbehind. Neither the alternative + PCRE2 does not allow \C to appear in lookbehind assertions (described + below) in UTF-8 or UTF-16 modes, because this would make it impossible + to calculate the length of the lookbehind. Neither the alternative matching function pcre2_dfa_match() nor the JIT optimizer support \C in these UTF modes. The former gives a match-time error; the latter fails to optimize and so the match is always run using the interpreter. - In the 32-bit library, however, \C is always supported (when not - explicitly locked out) because it always matches a single code unit, + In the 32-bit library, however, \C is always supported (when not ex- + plicitly locked out) because it always matches a single code unit, whether or not UTF-32 is specified. In general, the \C escape sequence is best avoided. However, one way of - using it that avoids the problem of malformed UTF-8 or UTF-16 charac- - ters is to use a lookahead to check the length of the next character, - as in this pattern, which could be used with a UTF-8 string (ignore + using it that avoids the problem of malformed UTF-8 or UTF-16 charac- + ters is to use a lookahead to check the length of the next character, + as in this pattern, which could be used with a UTF-8 string (ignore white space and line breaks): (?| (?=[\x00-\x7f])(\C) | @@ -7176,135 +7310,137 @@ MATCHING A SINGLE CODE UNIT (?=[\x{800}-\x{ffff}])(\C)(\C)(\C) | (?=[\x{10000}-\x{1fffff}])(\C)(\C)(\C)(\C)) - In this example, a group that starts with (?| resets the capturing - parentheses numbers in each alternative (see "Duplicate Group Numbers" + In this example, a group that starts with (?| resets the capturing + parentheses numbers in each alternative (see "Duplicate Group Numbers" below). The assertions at the start of each branch check the next UTF-8 - character for values whose encoding uses 1, 2, 3, or 4 bytes, respec- - tively. The character's individual bytes are then captured by the - appropriate number of \C groups. + character for values whose encoding uses 1, 2, 3, or 4 bytes, respec- + tively. The character's individual bytes are then captured by the ap- + propriate number of \C groups. SQUARE BRACKETS AND CHARACTER CLASSES An opening square bracket introduces a character class, terminated by a closing square bracket. A closing square bracket on its own is not spe- - cial by default. If a closing square bracket is required as a member + cial by default. If a closing square bracket is required as a member of the class, it should be the first data character in the class (after - an initial circumflex, if present) or escaped with a backslash. This - means that, by default, an empty class cannot be defined. However, if - the PCRE2_ALLOW_EMPTY_CLASS option is set, a closing square bracket at + an initial circumflex, if present) or escaped with a backslash. This + means that, by default, an empty class cannot be defined. However, if + the PCRE2_ALLOW_EMPTY_CLASS option is set, a closing square bracket at the start does end the (empty) class. - A character class matches a single character in the subject. A matched + A character class matches a single character in the subject. A matched character must be in the set of characters defined by the class, unless - the first character in the class definition is a circumflex, in which + the first character in the class definition is a circumflex, in which case the subject character must not be in the set defined by the class. - If a circumflex is actually required as a member of the class, ensure + If a circumflex is actually required as a member of the class, ensure it is not the first character, or escape it with a backslash. - For example, the character class [aeiou] matches any lower case vowel, - while [^aeiou] matches any character that is not a lower case vowel. + For example, the character class [aeiou] matches any lower case vowel, + while [^aeiou] matches any character that is not a lower case vowel. Note that a circumflex is just a convenient notation for specifying the - characters that are in the class by enumerating those that are not. A - class that starts with a circumflex is not an assertion; it still con- - sumes a character from the subject string, and therefore it fails if + characters that are in the class by enumerating those that are not. A + class that starts with a circumflex is not an assertion; it still con- + sumes a character from the subject string, and therefore it fails if the current pointer is at the end of the string. - Characters in a class may be specified by their code points using \o, - \x, or \N{U+hh..} in the usual way. When caseless matching is set, any - letters in a class represent both their upper case and lower case ver- - sions, so for example, a caseless [aeiou] matches "A" as well as "a", - and a caseless [^aeiou] does not match "A", whereas a caseful version - would. + Characters in a class may be specified by their code points using \o, + \x, or \N{U+hh..} in the usual way. When caseless matching is set, any + letters in a class represent both their upper case and lower case ver- + sions, so for example, a caseless [aeiou] matches "A" as well as "a", + and a caseless [^aeiou] does not match "A", whereas a caseful version + would. Note that there are two ASCII characters, K and S, that, in ad- + dition to their lower case ASCII equivalents, are case-equivalent with + Unicode U+212A (Kelvin sign) and U+017F (long S) respectively when ei- + ther PCRE2_UTF or PCRE2_UCP is set. Characters that might indicate line breaks are never treated in any - special way when matching character classes, whatever line-ending - sequence is in use, and whatever setting of the PCRE2_DOTALL and + special way when matching character classes, whatever line-ending se- + quence is in use, and whatever setting of the PCRE2_DOTALL and PCRE2_MULTILINE options is used. A class such as [^a] always matches one of these characters. The generic character type escape sequences \d, \D, \h, \H, \p, \P, \s, \S, \v, \V, \w, and \W may appear in a character class, and add the characters that they match to the class. For example, [\dABCDEF] - matches any hexadecimal digit. In UTF modes, the PCRE2_UCP option - affects the meanings of \d, \s, \w and their upper case partners, just - as it does when they appear outside a character class, as described in - the section entitled "Generic character types" above. The escape - sequence \b has a different meaning inside a character class; it - matches the backspace character. The sequences \B, \R, and \X are not - special inside a character class. Like any other unrecognized escape - sequences, they cause an error. The same is true for \N when not fol- - lowed by an opening brace. + matches any hexadecimal digit. In UTF modes, the PCRE2_UCP option af- + fects the meanings of \d, \s, \w and their upper case partners, just as + it does when they appear outside a character class, as described in the + section entitled "Generic character types" above. The escape sequence + \b has a different meaning inside a character class; it matches the + backspace character. The sequences \B, \R, and \X are not special in- + side a character class. Like any other unrecognized escape sequences, + they cause an error. The same is true for \N when not followed by an + opening brace. The minus (hyphen) character can be used to specify a range of charac- - ters in a character class. For example, [d-m] matches any letter - between d and m, inclusive. If a minus character is required in a - class, it must be escaped with a backslash or appear in a position - where it cannot be interpreted as indicating a range, typically as the - first or last character in the class, or immediately after a range. For - example, [b-d-z] matches letters in the range b to d, a hyphen charac- - ter, or z. + ters in a character class. For example, [d-m] matches any letter be- + tween d and m, inclusive. If a minus character is required in a class, + it must be escaped with a backslash or appear in a position where it + cannot be interpreted as indicating a range, typically as the first or + last character in the class, or immediately after a range. For example, + [b-d-z] matches letters in the range b to d, a hyphen character, or z. Perl treats a hyphen as a literal if it appears before or after a POSIX class (see below) or before or after a character type escape such as as - \d or \H. However, unless the hyphen is the last character in the - class, Perl outputs a warning in its warning mode, as this is most - likely a user error. As PCRE2 has no facility for warning, an error is + \d or \H. However, unless the hyphen is the last character in the + class, Perl outputs a warning in its warning mode, as this is most + likely a user error. As PCRE2 has no facility for warning, an error is given in these cases. It is not possible to have the literal character "]" as the end charac- - ter of a range. A pattern such as [W-]46] is interpreted as a class of - two characters ("W" and "-") followed by a literal string "46]", so it - would match "W46]" or "-46]". However, if the "]" is escaped with a - backslash it is interpreted as the end of range, so [W-\]46] is inter- - preted as a class containing a range followed by two other characters. - The octal or hexadecimal representation of "]" can also be used to end + ter of a range. A pattern such as [W-]46] is interpreted as a class of + two characters ("W" and "-") followed by a literal string "46]", so it + would match "W46]" or "-46]". However, if the "]" is escaped with a + backslash it is interpreted as the end of range, so [W-\]46] is inter- + preted as a class containing a range followed by two other characters. + The octal or hexadecimal representation of "]" can also be used to end a range. Ranges normally include all code points between the start and end char- - acters, inclusive. They can also be used for code points specified - numerically, for example [\000-\037]. Ranges can include any characters - that are valid for the current mode. In any UTF mode, the so-called - "surrogate" characters (those whose code points lie between 0xd800 and - 0xdfff inclusive) may not be specified explicitly by default (the - PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES option disables this check). How- + acters, inclusive. They can also be used for code points specified nu- + merically, for example [\000-\037]. Ranges can include any characters + that are valid for the current mode. In any UTF mode, the so-called + "surrogate" characters (those whose code points lie between 0xd800 and + 0xdfff inclusive) may not be specified explicitly by default (the + PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES option disables this check). How- ever, ranges such as [\x{d7ff}-\x{e000}], which include the surrogates, are always permitted. - There is a special case in EBCDIC environments for ranges whose end + There is a special case in EBCDIC environments for ranges whose end points are both specified as literal letters in the same case. For com- - patibility with Perl, EBCDIC code points within the range that are not - letters are omitted. For example, [h-k] matches only four characters, + patibility with Perl, EBCDIC code points within the range that are not + letters are omitted. For example, [h-k] matches only four characters, even though the codes for h and k are 0x88 and 0x92, a range of 11 code - points. However, if the range is specified numerically, for example, + points. However, if the range is specified numerically, for example, [\x88-\x92] or [h-\x92], all code points are included. If a range that includes letters is used when caseless matching is set, it matches the letters in either case. For example, [W-c] is equivalent - to [][\\^_`wxyzabc], matched caselessly, and in a non-UTF mode, if - character tables for a French locale are in use, [\xc8-\xcb] matches + to [][\\^_`wxyzabc], matched caselessly, and in a non-UTF mode, if + character tables for a French locale are in use, [\xc8-\xcb] matches accented E characters in both cases. - A circumflex can conveniently be used with the upper case character - types to specify a more restricted set of characters than the matching - lower case type. For example, the class [^\W_] matches any letter or + A circumflex can conveniently be used with the upper case character + types to specify a more restricted set of characters than the matching + lower case type. For example, the class [^\W_] matches any letter or digit, but not underscore, whereas [\w] includes underscore. A positive character class should be read as "something OR something OR ..." and a negative class as "NOT something AND NOT something AND NOT ...". - The only metacharacters that are recognized in character classes are - backslash, hyphen (only where it can be interpreted as specifying a - range), circumflex (only at the start), opening square bracket (only - when it can be interpreted as introducing a POSIX class name, or for a - special compatibility feature - see the next two sections), and the - terminating closing square bracket. However, escaping other non- - alphanumeric characters does no harm. + The only metacharacters that are recognized in character classes are + backslash, hyphen (only where it can be interpreted as specifying a + range), circumflex (only at the start), opening square bracket (only + when it can be interpreted as introducing a POSIX class name, or for a + special compatibility feature - see the next two sections), and the + terminating closing square bracket. However, escaping other non-al- + phanumeric characters does no harm. POSIX CHARACTER CLASSES Perl supports the POSIX notation for character classes. This uses names - enclosed by [: and :] within the enclosing square brackets. PCRE2 also + enclosed by [: and :] within the enclosing square brackets. PCRE2 also supports this notation. For example, [01[:alpha:]%] @@ -7327,13 +7463,13 @@ POSIX CHARACTER CLASSES word "word" characters (same as \w) xdigit hexadecimal digits - The default "space" characters are HT (9), LF (10), VT (11), FF (12), - CR (13), and space (32). If locale-specific matching is taking place, - the list of space characters may be different; there may be fewer or + The default "space" characters are HT (9), LF (10), VT (11), FF (12), + CR (13), and space (32). If locale-specific matching is taking place, + the list of space characters may be different; there may be fewer or more of them. "Space" and \s match the same set of characters. - The name "word" is a Perl extension, and "blank" is a GNU extension - from Perl 5.8. Another Perl extension is negation, which is indicated + The name "word" is a Perl extension, and "blank" is a GNU extension + from Perl 5.8. Another Perl extension is negation, which is indicated by a ^ character after the colon. For example, [12[:^digit:]] @@ -7344,11 +7480,11 @@ POSIX CHARACTER CLASSES By default, characters with values greater than 127 do not match any of the POSIX character classes, although this may be different for charac- - ters in the range 128-255 when locale-specific matching is happening. - However, if the PCRE2_UCP option is passed to pcre2_compile(), some of - the classes are changed so that Unicode character properties are used. - This is achieved by replacing certain POSIX classes with other - sequences, as follows: + ters in the range 128-255 when locale-specific matching is happening. + However, if the PCRE2_UCP option is passed to pcre2_compile(), some of + the classes are changed so that Unicode character properties are used. + This is achieved by replacing certain POSIX classes with other se- + quences, as follows: [:alnum:] becomes \p{Xan} [:alpha:] becomes \p{L} @@ -7360,10 +7496,10 @@ POSIX CHARACTER CLASSES [:upper:] becomes \p{Lu} [:word:] becomes \p{Xwd} - Negated versions, such as [:^alpha:] use \P instead of \p. Three other + Negated versions, such as [:^alpha:] use \P instead of \p. Three other POSIX classes are handled specially in UCP mode: - [:graph:] This matches characters that have glyphs that mark the page + [:graph:] This matches characters that have glyphs that mark the page when printed. In Unicode property terms, it matches all char- acters with the L, M, N, P, S, or Cf properties, except for: @@ -7372,60 +7508,60 @@ POSIX CHARACTER CLASSES U+2066 - U+2069 Various "isolate"s - [:print:] This matches the same characters as [:graph:] plus space - characters that are not controls, that is, characters with + [:print:] This matches the same characters as [:graph:] plus space + characters that are not controls, that is, characters with the Zs property. [:punct:] This matches all characters that have the Unicode P (punctua- - tion) property, plus those characters with code points less + tion) property, plus those characters with code points less than 256 that have the S (Symbol) property. - The other POSIX classes are unchanged, and match only characters with + The other POSIX classes are unchanged, and match only characters with code points less than 256. COMPATIBILITY FEATURE FOR WORD BOUNDARIES - In the POSIX.2 compliant library that was included in 4.4BSD Unix, the - ugly syntax [[:<:]] and [[:>:]] is used for matching "start of word" + In the POSIX.2 compliant library that was included in 4.4BSD Unix, the + ugly syntax [[:<:]] and [[:>:]] is used for matching "start of word" and "end of word". PCRE2 treats these items as follows: [[:<:]] is converted to \b(?=\w) [[:>:]] is converted to \b(?<=\w) Only these exact character sequences are recognized. A sequence such as - [a[:<:]b] provokes error for an unrecognized POSIX class name. This - support is not compatible with Perl. It is provided to help migrations + [a[:<:]b] provokes error for an unrecognized POSIX class name. This + support is not compatible with Perl. It is provided to help migrations from other environments, and is best not used in any new patterns. Note - that \b matches at the start and the end of a word (see "Simple asser- - tions" above), and in a Perl-style pattern the preceding or following - character normally shows which is wanted, without the need for the - assertions that are used above in order to give exactly the POSIX be- - haviour. + that \b matches at the start and the end of a word (see "Simple asser- + tions" above), and in a Perl-style pattern the preceding or following + character normally shows which is wanted, without the need for the as- + sertions that are used above in order to give exactly the POSIX behav- + iour. VERTICAL BAR - Vertical bar characters are used to separate alternative patterns. For + Vertical bar characters are used to separate alternative patterns. For example, the pattern gilbert|sullivan - matches either "gilbert" or "sullivan". Any number of alternatives may - appear, and an empty alternative is permitted (matching the empty + matches either "gilbert" or "sullivan". Any number of alternatives may + appear, and an empty alternative is permitted (matching the empty string). The matching process tries each alternative in turn, from left - to right, and the first one that succeeds is used. If the alternatives - are within a group (defined below), "succeeds" means matching the rest + to right, and the first one that succeeds is used. If the alternatives + are within a group (defined below), "succeeds" means matching the rest of the main pattern as well as the alternative in the group. INTERNAL OPTION SETTING - The settings of the PCRE2_CASELESS, PCRE2_MULTILINE, PCRE2_DOTALL, - PCRE2_EXTENDED, PCRE2_EXTENDED_MORE, and PCRE2_NO_AUTO_CAPTURE options - can be changed from within the pattern by a sequence of letters - enclosed between "(?" and ")". These options are Perl-compatible, and - are described in detail in the pcre2api documentation. The option let- + The settings of the PCRE2_CASELESS, PCRE2_MULTILINE, PCRE2_DOTALL, + PCRE2_EXTENDED, PCRE2_EXTENDED_MORE, and PCRE2_NO_AUTO_CAPTURE options + can be changed from within the pattern by a sequence of letters en- + closed between "(?" and ")". These options are Perl-compatible, and + are described in detail in the pcre2api documentation. The option let- ters are: i for PCRE2_CASELESS @@ -7436,49 +7572,49 @@ INTERNAL OPTION SETTING xx for PCRE2_EXTENDED_MORE For example, (?im) sets caseless, multiline matching. It is also possi- - ble to unset these options by preceding the relevant letters with a - hyphen, for example (?-im). The two "extended" options are not indepen- + ble to unset these options by preceding the relevant letters with a hy- + phen, for example (?-im). The two "extended" options are not indepen- dent; unsetting either one cancels the effects of both of them. - A combined setting and unsetting such as (?im-sx), which sets - PCRE2_CASELESS and PCRE2_MULTILINE while unsetting PCRE2_DOTALL and - PCRE2_EXTENDED, is also permitted. Only one hyphen may appear in the - options string. If a letter appears both before and after the hyphen, - the option is unset. An empty options setting "(?)" is allowed. Need- + A combined setting and unsetting such as (?im-sx), which sets + PCRE2_CASELESS and PCRE2_MULTILINE while unsetting PCRE2_DOTALL and + PCRE2_EXTENDED, is also permitted. Only one hyphen may appear in the + options string. If a letter appears both before and after the hyphen, + the option is unset. An empty options setting "(?)" is allowed. Need- less to say, it has no effect. - If the first character following (? is a circumflex, it causes all of - the above options to be unset. Thus, (?^) is equivalent to (?-imnsx). - Letters may follow the circumflex to cause some options to be re- - instated, but a hyphen may not appear. + If the first character following (? is a circumflex, it causes all of + the above options to be unset. Thus, (?^) is equivalent to (?-imnsx). + Letters may follow the circumflex to cause some options to be re-in- + stated, but a hyphen may not appear. - The PCRE2-specific options PCRE2_DUPNAMES and PCRE2_UNGREEDY can be - changed in the same way as the Perl-compatible options by using the + The PCRE2-specific options PCRE2_DUPNAMES and PCRE2_UNGREEDY can be + changed in the same way as the Perl-compatible options by using the characters J and U respectively. However, these are not unset by (?^). - When one of these option changes occurs at top level (that is, not - inside group parentheses), the change applies to the remainder of the - pattern that follows. An option change within a group (see below for a + When one of these option changes occurs at top level (that is, not in- + side group parentheses), the change applies to the remainder of the + pattern that follows. An option change within a group (see below for a description of groups) affects only that part of the group that follows it, so (a(?i)b)c - matches abc and aBc and no other strings (assuming PCRE2_CASELESS is - not used). By this means, options can be made to have different set- + matches abc and aBc and no other strings (assuming PCRE2_CASELESS is + not used). By this means, options can be made to have different set- tings in different parts of the pattern. Any changes made in one alter- - native do carry on into subsequent branches within the same group. For + native do carry on into subsequent branches within the same group. For example, (a(?i)b|c) - matches "ab", "aB", "c", and "C", even though when matching "C" the - first branch is abandoned before the option setting. This is because - the effects of option settings happen at compile time. There would be + matches "ab", "aB", "c", and "C", even though when matching "C" the + first branch is abandoned before the option setting. This is because + the effects of option settings happen at compile time. There would be some very weird behaviour otherwise. - As a convenient shorthand, if any option settings are required at the - start of a non-capturing group (see the next section), the option let- + As a convenient shorthand, if any option settings are required at the + start of a non-capturing group (see the next section), the option let- ters may appear between the "?" and the ":". Thus the two patterns (?i:saturday|sunday) @@ -7486,39 +7622,39 @@ INTERNAL OPTION SETTING match exactly the same set of strings. - Note: There are other PCRE2-specific options, applying to the whole - pattern, which can be set by the application when the compiling func- - tion is called. In addition, the pattern can contain special leading - sequences such as (*CRLF) to override what the application has set or - what has been defaulted. Details are given in the section entitled + Note: There are other PCRE2-specific options, applying to the whole + pattern, which can be set by the application when the compiling func- + tion is called. In addition, the pattern can contain special leading + sequences such as (*CRLF) to override what the application has set or + what has been defaulted. Details are given in the section entitled "Newline sequences" above. There are also the (*UTF) and (*UCP) leading - sequences that can be used to set UTF and Unicode property modes; they - are equivalent to setting the PCRE2_UTF and PCRE2_UCP options, respec- - tively. However, the application can set the PCRE2_NEVER_UTF and - PCRE2_NEVER_UCP options, which lock out the use of the (*UTF) and + sequences that can be used to set UTF and Unicode property modes; they + are equivalent to setting the PCRE2_UTF and PCRE2_UCP options, respec- + tively. However, the application can set the PCRE2_NEVER_UTF and + PCRE2_NEVER_UCP options, which lock out the use of the (*UTF) and (*UCP) sequences. GROUPS - Groups are delimited by parentheses (round brackets), which can be + Groups are delimited by parentheses (round brackets), which can be nested. Turning part of a pattern into a group does two things: 1. It localizes a set of alternatives. For example, the pattern cat(aract|erpillar|) - matches "cataract", "caterpillar", or "cat". Without the parentheses, + matches "cataract", "caterpillar", or "cat". Without the parentheses, it would match "cataract", "erpillar" or an empty string. - 2. It creates a "capture group". This means that, when the whole pat- - tern matches, the portion of the subject string that matched the group - is passed back to the caller, separately from the portion that matched - the whole pattern. (This applies only to the traditional matching + 2. It creates a "capture group". This means that, when the whole pat- + tern matches, the portion of the subject string that matched the group + is passed back to the caller, separately from the portion that matched + the whole pattern. (This applies only to the traditional matching function; the DFA matching function does not support capturing.) Opening parentheses are counted from left to right (starting from 1) to - obtain numbers for capture groups. For example, if the string "the red + obtain numbers for capture groups. For example, if the string "the red king" is matched against the pattern the ((red|white) (king|queen)) @@ -7526,11 +7662,11 @@ GROUPS the captured substrings are "red king", "red", and "king", and are num- bered 1, 2, and 3, respectively. - The fact that plain parentheses fulfil two functions is not always - helpful. There are often times when grouping is required without cap- - turing. If an opening parenthesis is followed by a question mark and a - colon, the group does not do any capturing, and is not counted when - computing the number of any subsequent capture groups. For example, if + The fact that plain parentheses fulfil two functions is not always + helpful. There are often times when grouping is required without cap- + turing. If an opening parenthesis is followed by a question mark and a + colon, the group does not do any capturing, and is not counted when + computing the number of any subsequent capture groups. For example, if the string "the white queen" is matched against the pattern the ((?:red|white) (king|queen)) @@ -7538,16 +7674,16 @@ GROUPS the captured substrings are "white queen" and "queen", and are numbered 1 and 2. The maximum number of capture groups is 65535. - As a convenient shorthand, if any option settings are required at the - start of a non-capturing group, the option letters may appear between + As a convenient shorthand, if any option settings are required at the + start of a non-capturing group, the option letters may appear between the "?" and the ":". Thus the two patterns (?i:saturday|sunday) (?:(?i)saturday|sunday) match exactly the same set of strings. Because alternative branches are - tried from left to right, and options are not reset until the end of - the group is reached, an option setting in one branch does affect sub- + tried from left to right, and options are not reset until the end of + the group is reached, an option setting in one branch does affect sub- sequent branches, so the above patterns match "SUNDAY" as well as "Sat- urday". @@ -7555,19 +7691,19 @@ GROUPS DUPLICATE GROUP NUMBERS Perl 5.10 introduced a feature whereby each alternative in a group uses - the same numbers for its capturing parentheses. Such a group starts - with (?| and is itself a non-capturing group. For example, consider + the same numbers for its capturing parentheses. Such a group starts + with (?| and is itself a non-capturing group. For example, consider this pattern: (?|(Sat)ur|(Sun))day - Because the two alternatives are inside a (?| group, both sets of cap- - turing parentheses are numbered one. Thus, when the pattern matches, - you can look at captured substring number one, whichever alternative - matched. This construct is useful when you want to capture part, but + Because the two alternatives are inside a (?| group, both sets of cap- + turing parentheses are numbered one. Thus, when the pattern matches, + you can look at captured substring number one, whichever alternative + matched. This construct is useful when you want to capture part, but not all, of one of a number of alternatives. Inside a (?| group, paren- - theses are numbered as usual, but the number is reset at the start of - each branch. The numbers of any capturing parentheses that follow the + theses are numbered as usual, but the number is reset at the start of + each branch. The numbers of any capturing parentheses that follow the whole group start after the highest number used in any branch. The fol- lowing example is taken from the Perl documentation. The numbers under- neath show in which buffer the captured content will be stored. @@ -7576,13 +7712,13 @@ DUPLICATE GROUP NUMBERS / ( a ) (?| x ( y ) z | (p (q) r) | (t) u (v) ) ( z ) /x # 1 2 2 3 2 3 4 - A backreference to a capture group uses the most recent value that is + A backreference to a capture group uses the most recent value that is set for the group. The following pattern matches "abcabc" or "defdef": /(?|(abc)|(def))\1/ - In contrast, a subroutine call to a capture group always refers to the - first one in the pattern with the given number. The following pattern + In contrast, a subroutine call to a capture group always refers to the + first one in the pattern with the given number. The following pattern matches "abcabc" or "defabc": /(?|(abc)|(def))(?1)/ @@ -7593,24 +7729,24 @@ DUPLICATE GROUP NUMBERS If a condition test for a group's having matched refers to a non-unique number, the test is true if any group with that number has matched. - An alternative approach to using this "branch reset" feature is to use + An alternative approach to using this "branch reset" feature is to use duplicate named groups, as described in the next section. NAMED CAPTURE GROUPS Identifying capture groups by number is simple, but it can be very hard - to keep track of the numbers in complicated patterns. Furthermore, if - an expression is modified, the numbers may change. To help with this - difficulty, PCRE2 supports the naming of capture groups. This feature - was not added to Perl until release 5.10. Python had the feature ear- - lier, and PCRE1 introduced it at release 4.0, using the Python syntax. + to keep track of the numbers in complicated patterns. Furthermore, if + an expression is modified, the numbers may change. To help with this + difficulty, PCRE2 supports the naming of capture groups. This feature + was not added to Perl until release 5.10. Python had the feature ear- + lier, and PCRE1 introduced it at release 4.0, using the Python syntax. PCRE2 supports both the Perl and the Python syntax. - In PCRE2, a capture group can be named in one of three ways: + In PCRE2, a capture group can be named in one of three ways: (?<name>...) or (?'name'...) as in Perl, or (?P<name>...) as in Python. - Names may be up to 32 code units long. When PCRE2_UTF is not set, they - may contain only ASCII alphanumeric characters and underscores, but + Names may be up to 32 code units long. When PCRE2_UTF is not set, they + may contain only ASCII alphanumeric characters and underscores, but must start with a non-digit. When PCRE2_UTF is set, the syntax of group names is extended to allow any Unicode letter or Unicode decimal digit. In other words, group names must match one of these patterns: @@ -7618,74 +7754,77 @@ NAMED CAPTURE GROUPS ^[_A-Za-z][_A-Za-z0-9]*\z when PCRE2_UTF is not set ^[_\p{L}][_\p{L}\p{Nd}]*\z when PCRE2_UTF is set - References to capture groups from other parts of the pattern, such as - backreferences, recursion, and conditions, can all be made by name as + References to capture groups from other parts of the pattern, such as + backreferences, recursion, and conditions, can all be made by name as well as by number. Named capture groups are allocated numbers as well as names, exactly as - if the names were not present. In both PCRE2 and Perl, capture groups - are primarily identified by numbers; any names are just aliases for + if the names were not present. In both PCRE2 and Perl, capture groups + are primarily identified by numbers; any names are just aliases for these numbers. The PCRE2 API provides function calls for extracting the - complete name-to-number translation table from a compiled pattern, as - well as convenience functions for extracting captured substrings by + complete name-to-number translation table from a compiled pattern, as + well as convenience functions for extracting captured substrings by name. - Warning: When more than one capture group has the same number, as - described in the previous section, a name given to one of them applies - to all of them. Perl allows identically numbered groups to have differ- - ent names. Consider this pattern, where there are two capture groups, - both numbered 1: + Warning: When more than one capture group has the same number, as de- + scribed in the previous section, a name given to one of them applies to + all of them. Perl allows identically numbered groups to have different + names. Consider this pattern, where there are two capture groups, both + numbered 1: (?|(?<AA>aa)|(?<BB>bb)) - Perl allows this, with both names AA and BB as aliases of group 1. + Perl allows this, with both names AA and BB as aliases of group 1. Thus, after a successful match, both names yield the same value (either "aa" or "bb"). - In an attempt to reduce confusion, PCRE2 does not allow the same group + In an attempt to reduce confusion, PCRE2 does not allow the same group number to be associated with more than one name. The example above pro- - vokes a compile-time error. However, there is still scope for confu- + vokes a compile-time error. However, there is still scope for confu- sion. Consider this pattern: (?|(?<AA>aa)|(bb)) Although the second group number 1 is not explicitly named, the name AA - is still an alias for any group 1. Whether the pattern matches "aa" or + is still an alias for any group 1. Whether the pattern matches "aa" or "bb", a reference by name to group AA yields the matched string. - By default, a name must be unique within a pattern, except that dupli- + By default, a name must be unique within a pattern, except that dupli- cate names are permitted for groups with the same number, for example: (?|(?<AA>aa)|(?<AA>bb)) The duplicate name constraint can be disabled by setting the PCRE2_DUP- - NAMES option at compile time, or by the use of (?J) within the pattern. - Duplicate names can be useful for patterns where only one instance of - the named capture group can match. Suppose you want to match the name - of a weekday, either as a 3-letter abbreviation or as the full name, - and in both cases you want to extract the abbreviation. This pattern + NAMES option at compile time, or by the use of (?J) within the pattern, + as described in the section entitled "Internal Option Setting" above. + + Duplicate names can be useful for patterns where only one instance of + the named capture group can match. Suppose you want to match the name + of a weekday, either as a 3-letter abbreviation or as the full name, + and in both cases you want to extract the abbreviation. This pattern (ignoring the line breaks) does the job: + (?J) (?<DN>Mon|Fri|Sun)(?:day)?| (?<DN>Tue)(?:sday)?| (?<DN>Wed)(?:nesday)?| (?<DN>Thu)(?:rsday)?| (?<DN>Sat)(?:urday)? - There are five capture groups, but only one is ever set after a match. - The convenience functions for extracting the data by name returns the - substring for the first (and in this example, the only) group of that + There are five capture groups, but only one is ever set after a match. + The convenience functions for extracting the data by name returns the + substring for the first (and in this example, the only) group of that name that matched. This saves searching to find which numbered group it - was. (An alternative way of solving this problem is to use a "branch + was. (An alternative way of solving this problem is to use a "branch reset" group, as described in the previous section.) - If you make a backreference to a non-unique named group from elsewhere - in the pattern, the groups to which the name refers are checked in the - order in which they appear in the overall pattern. The first one that - is set is used for the reference. For example, this pattern matches + If you make a backreference to a non-unique named group from elsewhere + in the pattern, the groups to which the name refers are checked in the + order in which they appear in the overall pattern. The first one that + is set is used for the reference. For example, this pattern matches both "foofoo" and "barbar" but not "foobar" or "barfoo": - (?:(?<n>foo)|(?<n>bar))\k<n> + (?J)(?:(?<n>foo)|(?<n>bar))\k<n> If you make a subroutine call to a non-unique named group, the one that @@ -7695,15 +7834,15 @@ NAMED CAPTURE GROUPS If you use a named reference in a condition test (see the section about conditions below), either to check whether a capture group has matched, or to check for recursion, all groups with the same name are tested. If - the condition is true for any one of them, the overall condition is - true. This is the same behaviour as testing by number. For further - details of the interfaces for handling named capture groups, see the + the condition is true for any one of them, the overall condition is + true. This is the same behaviour as testing by number. For further de- + tails of the interfaces for handling named capture groups, see the pcre2api documentation. REPETITION - Repetition is specified by quantifiers, which can follow any of the + Repetition is specified by quantifiers, which can follow any of the following items: a literal data character @@ -7714,20 +7853,20 @@ REPETITION an escape such as \d or \pL that matches a single character a character class a backreference - a parenthesized group (including most assertions) + a parenthesized group (including lookaround assertions) a subroutine call (recursive or otherwise) - The general repetition quantifier specifies a minimum and maximum num- - ber of permitted matches, by giving the two numbers in curly brackets - (braces), separated by a comma. The numbers must be less than 65536, + The general repetition quantifier specifies a minimum and maximum num- + ber of permitted matches, by giving the two numbers in curly brackets + (braces), separated by a comma. The numbers must be less than 65536, and the first must be less than or equal to the second. For example, z{2,4} - matches "zz", "zzz", or "zzzz". A closing brace on its own is not a - special character. If the second number is omitted, but the comma is - present, there is no upper limit; if the second number and the comma - are both omitted, the quantifier specifies an exact number of required + matches "zz", "zzz", or "zzzz". A closing brace on its own is not a + special character. If the second number is omitted, but the comma is + present, there is no upper limit; if the second number and the comma + are both omitted, the quantifier specifies an exact number of required matches. Thus [aeiou]{3,} @@ -7736,51 +7875,54 @@ REPETITION \d{8} - matches exactly 8 digits. An opening curly bracket that appears in a - position where a quantifier is not allowed, or one that does not match - the syntax of a quantifier, is taken as a literal character. For exam- + matches exactly 8 digits. An opening curly bracket that appears in a + position where a quantifier is not allowed, or one that does not match + the syntax of a quantifier, is taken as a literal character. For exam- ple, {,6} is not a quantifier, but a literal string of four characters. In UTF modes, quantifiers apply to characters rather than to individual - code units. Thus, for example, \x{100}{2} matches two characters, each + code units. Thus, for example, \x{100}{2} matches two characters, each of which is represented by a two-byte sequence in a UTF-8 string. Simi- - larly, \X{3} matches three Unicode extended grapheme clusters, each of - which may be several code units long (and they may be of different + larly, \X{3} matches three Unicode extended grapheme clusters, each of + which may be several code units long (and they may be of different lengths). The quantifier {0} is permitted, causing the expression to behave as if the previous item and the quantifier were not present. This may be use- - ful for capture groups that are referenced as subroutines from else- - where in the pattern (but see also the section entitled "Defining cap- + ful for capture groups that are referenced as subroutines from else- + where in the pattern (but see also the section entitled "Defining cap- ture groups for use by reference only" below). Except for parenthesized - groups, items that have a {0} quantifier are omitted from the compiled + groups, items that have a {0} quantifier are omitted from the compiled pattern. - For convenience, the three most common quantifiers have single-charac- + For convenience, the three most common quantifiers have single-charac- ter abbreviations: * is equivalent to {0,} + is equivalent to {1,} ? is equivalent to {0,1} - It is possible to construct infinite loops by following a group that - can match no characters with a quantifier that has no upper limit, for + It is possible to construct infinite loops by following a group that + can match no characters with a quantifier that has no upper limit, for example: (a?)* - Earlier versions of Perl and PCRE1 used to give an error at compile + Earlier versions of Perl and PCRE1 used to give an error at compile time for such patterns. However, because there are cases where this can - be useful, such patterns are now accepted, but if any repetition of the - group does in fact match no characters, the loop is forcibly broken. + be useful, such patterns are now accepted, but whenever an iteration of + such a group matches no characters, matching moves on to the next item + in the pattern instead of repeatedly matching an empty string. This + does not prevent backtracking into any of the iterations if a subse- + quent item fails to match. By default, quantifiers are "greedy", that is, they match as much as possible (up to the maximum number of permitted times), without causing the rest of the pattern to fail. The classic example of where this - gives problems is in trying to match comments in C programs. These - appear between /* and */ and within the comment, individual * and / - characters may appear. An attempt to match C comments by applying the - pattern + gives problems is in trying to match comments in C programs. These ap- + pear between /* and */ and within the comment, individual * and / char- + acters may appear. An attempt to match C comments by applying the pat- + tern /\*.*\*/ @@ -7812,16 +7954,16 @@ REPETITION words, it inverts the default behaviour. When a parenthesized group is quantified with a minimum repeat count - that is greater than 1 or with a limited maximum, more memory is - required for the compiled pattern, in proportion to the size of the - minimum or maximum. + that is greater than 1 or with a limited maximum, more memory is re- + quired for the compiled pattern, in proportion to the size of the mini- + mum or maximum. If a pattern starts with .* or .{0,} and the PCRE2_DOTALL option (equivalent to Perl's /s) is set, thus allowing the dot to match new- lines, the pattern is implicitly anchored, because whatever follows will be tried against every character position in the subject string, - so there is no point in retrying the overall match at any position - after the first. PCRE2 normally treats such a pattern as though it were + so there is no point in retrying the overall match at any position af- + ter the first. PCRE2 normally treats such a pattern as though it were preceded by \A. In cases where it is known that the subject string contains no new- @@ -7906,8 +8048,8 @@ ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS match, if anchored at the current point in the subject string. Atomic groups are not capture groups. Simple cases such as the above - example can be thought of as a maximizing repeat that must swallow - everything it can. So, while both \d+ and \d+? are prepared to adjust + example can be thought of as a maximizing repeat that must swallow ev- + erything it can. So, while both \d+ and \d+? are prepared to adjust the number of digits they match in order to make the rest of the pat- tern match, (?>\d+) can only match an entire sequence of digits. @@ -7925,10 +8067,10 @@ ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS (abc|xyz){2,3}+ - Possessive quantifiers are always greedy; the setting of the - PCRE2_UNGREEDY option is ignored. They are a convenient notation for - the simpler forms of atomic group. However, there is no difference in - the meaning of a possessive quantifier and the equivalent atomic group, + Possessive quantifiers are always greedy; the setting of the PCRE2_UN- + GREEDY option is ignored. They are a convenient notation for the sim- + pler forms of atomic group. However, there is no difference in the + meaning of a possessive quantifier and the equivalent atomic group, though there may be a performance difference; possessive quantifiers should be slightly faster. @@ -7944,8 +8086,8 @@ ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS when B must follow. This feature can be disabled by the PCRE2_NO_AUTO- POSSESS option, or starting the pattern with (*NO_AUTO_POSSESS). - When a pattern contains an unlimited repeat inside a group that can - itself be repeated an unlimited number of times, the use of an atomic + When a pattern contains an unlimited repeat inside a group that can it- + self be repeated an unlimited number of times, the use of an atomic group is the only way to avoid some failing matches taking a very long time indeed. The pattern @@ -7959,13 +8101,13 @@ ATOMIC GROUPING AND POSSESSIVE QUANTIFIERS it takes a long time before reporting failure. This is because the string can be divided between the internal \D+ repeat and the external - * repeat in a large number of ways, and all have to be tried. (The - example uses [!?] rather than a single character at the end, because - both PCRE2 and Perl have an optimization that allows for fast failure - when a single character is used. They remember the last single charac- - ter that is required for a match, and fail early if it is not present - in the string.) If the pattern is changed so that it uses an atomic - group, like this: + * repeat in a large number of ways, and all have to be tried. (The ex- + ample uses [!?] rather than a single character at the end, because both + PCRE2 and Perl have an optimization that allows for fast failure when a + single character is used. They remember the last single character that + is required for a match, and fail early if it is not present in the + string.) If the pattern is changed so that it uses an atomic group, + like this: ((?>\D+)|<\d+>)*[!?] @@ -8075,8 +8217,8 @@ BACKREFERENCES A backreference that occurs inside the group to which it refers fails when the group is first used, so, for example, (a\1) never matches. - However, such references can be useful inside repeated groups. For - example, the pattern + However, such references can be useful inside repeated groups. For ex- + ample, the pattern (a|b\1)+ @@ -8087,10 +8229,10 @@ BACKREFERENCES the backreference. This can be done using alternation, as in the exam- ple above, or by a quantifier with a minimum of zero. - Backreferences of this type cause the group that they reference to be - treated as an atomic group. Once the whole group has been matched, a - subsequent matching failure cannot cause backtracking into the middle - of the group. + For versions of PCRE2 less than 10.25, backreferences of this type used + to cause the group that they reference to be treated as an atomic + group. This restriction no longer applies, and backtracking into such + groups can occur as normal. ASSERTIONS @@ -8102,66 +8244,63 @@ ASSERTIONS More complicated assertions are coded as parenthesized groups. There are two kinds: those that look ahead of the current position in the - subject string, and those that look behind it, and in each case an - assertion may be positive (must match for the assertion to be true) or + subject string, and those that look behind it, and in each case an as- + sertion may be positive (must match for the assertion to be true) or negative (must not match for the assertion to be true). An assertion group is matched in the normal way, and if it is true, matching contin- - ues after it, but with the matching position in the subject string is - was it was before the assertion was processed. - - A lookaround assertion may also appear as the condition in a condi- - tional group (see below). In this case, the result of matching the - assertion determines which branch of the condition is followed. - - Assertion groups are not capture groups. If an assertion contains cap- - ture groups within it, these are counted for the purposes of numbering - the capture groups in the whole pattern. Within each branch of an - assertion, locally captured substrings may be referenced in the usual - way. For example, a sequence such as (.)\g{-1} can be used to check + ues after it, but with the matching position in the subject string re- + set to what it was before the assertion was processed. + + The Perl-compatible lookaround assertions are atomic. If an assertion + is true, but there is a subsequent matching failure, there is no back- + tracking into the assertion. However, there are some cases where non- + atomic assertions can be useful. PCRE2 has some support for these, de- + scribed in the section entitled "Non-atomic assertions" below, but they + are not Perl-compatible. + + A lookaround assertion may appear as the condition in a conditional + group (see below). In this case, the result of matching the assertion + determines which branch of the condition is followed. + + Assertion groups are not capture groups. If an assertion contains cap- + ture groups within it, these are counted for the purposes of numbering + the capture groups in the whole pattern. Within each branch of an as- + sertion, locally captured substrings may be referenced in the usual + way. For example, a sequence such as (.)\g{-1} can be used to check that two adjacent characters are the same. - When a branch within an assertion fails to match, any substrings that - were captured are discarded (as happens with any pattern branch that - fails to match). A negative assertion is true only when all its + When a branch within an assertion fails to match, any substrings that + were captured are discarded (as happens with any pattern branch that + fails to match). A negative assertion is true only when all its branches fail to match; this means that no captured substrings are ever - retained after a successful negative assertion. When an assertion con- + retained after a successful negative assertion. When an assertion con- tains a matching branch, what happens depends on the type of assertion. - For a positive assertion, internally captured substrings in the suc- - cessful branch are retained, and matching continues with the next pat- - tern item after the assertion. For a negative assertion, a matching - branch means that the assertion is not true. If such an assertion is - being used as a condition in a conditional group (see below), captured - substrings are retained, because matching continues with the "no" + For a positive assertion, internally captured substrings in the suc- + cessful branch are retained, and matching continues with the next pat- + tern item after the assertion. For a negative assertion, a matching + branch means that the assertion is not true. If such an assertion is + being used as a condition in a conditional group (see below), captured + substrings are retained, because matching continues with the "no" branch of the condition. For other failing negative assertions, control passes to the previous backtracking point, thus discarding any captured strings within the assertion. - For compatibility with Perl, most assertion groups may be repeated; - though it makes no sense to assert the same thing several times, the - side effect of capturing may occasionally be useful. However, an asser- - tion that forms the condition for a conditional group may not be quan- - tified. In practice, for other assertions, there only three cases: - - (1) If the quantifier is {0}, the assertion is never obeyed during - matching. However, it may contain internal capture groups that are - called from elsewhere via the subroutine mechanism. - - (2) If quantifier is {0,n} where n is greater than zero, it is treated - as if it were {0,1}. At run time, the rest of the pattern match is - tried with and without the assertion, the order depending on the greed- - iness of the quantifier. - - (3) If the minimum repetition is greater than zero, the quantifier is - ignored. The assertion is obeyed just once when encountered during - matching. + Most assertion groups may be repeated; though it makes no sense to as- + sert the same thing several times, the side effect of capturing in pos- + itive assertions may occasionally be useful. However, an assertion that + forms the condition for a conditional group may not be quantified. + PCRE2 used to restrict the repetition of assertions, but from release + 10.35 the only restriction is that an unlimited maximum repetition is + changed to be one more than the minimum. For example, {3,} is treated + as {3,4}. Alphabetic assertion names - Traditionally, symbolic sequences such as (?= and (?<= have been used - to specify lookaround assertions. Perl 5.28 introduced some experimen- + Traditionally, symbolic sequences such as (?= and (?<= have been used + to specify lookaround assertions. Perl 5.28 introduced some experimen- tal alphabetic alternatives which might be easier to remember. They all - start with (* instead of (? and must be written using lower case let- + start with (* instead of (? and must be written using lower case let- ters. PCRE2 supports the following synonyms: (*positive_lookahead: or (*pla: is the same as (?= @@ -8169,8 +8308,8 @@ ASSERTIONS (*positive_lookbehind: or (*plb: is the same as (?<= (*negative_lookbehind: or (*nlb: is the same as (?<! - For example, (*pla:foo) is the same assertion as (?=foo). In the fol- - lowing sections, the various assertions are described using the origi- + For example, (*pla:foo) is the same assertion as (?=foo). In the fol- + lowing sections, the various assertions are described using the origi- nal symbolic forms. Lookahead assertions @@ -8180,38 +8319,38 @@ ASSERTIONS \w+(?=;) - matches a word followed by a semicolon, but does not include the semi- + matches a word followed by a semicolon, but does not include the semi- colon in the match, and foo(?!bar) - matches any occurrence of "foo" that is not followed by "bar". Note + matches any occurrence of "foo" that is not followed by "bar". Note that the apparently similar pattern (?!foo)bar - does not find an occurrence of "bar" that is preceded by something - other than "foo"; it finds any occurrence of "bar" whatsoever, because + does not find an occurrence of "bar" that is preceded by something + other than "foo"; it finds any occurrence of "bar" whatsoever, because the assertion (?!foo) is always true when the next three characters are "bar". A lookbehind assertion is needed to achieve the other effect. If you want to force a matching failure at some point in a pattern, the - most convenient way to do it is with (?!) because an empty string - always matches, so an assertion that requires there not to be an empty + most convenient way to do it is with (?!) because an empty string al- + ways matches, so an assertion that requires there not to be an empty string must always fail. The backtracking control verb (*FAIL) or (*F) is a synonym for (?!). Lookbehind assertions - Lookbehind assertions start with (?<= for positive assertions and (?<! + Lookbehind assertions start with (?<= for positive assertions and (?<! for negative assertions. For example, (?<!foo)bar - does find an occurrence of "bar" that is not preceded by "foo". The - contents of a lookbehind assertion are restricted such that all the + does find an occurrence of "bar" that is not preceded by "foo". The + contents of a lookbehind assertion are restricted such that all the strings it matches must have a fixed length. However, if there are sev- - eral top-level alternatives, they do not all have to have the same + eral top-level alternatives, they do not all have to have the same fixed length. Thus (?<=bullock|donkey) @@ -8220,74 +8359,74 @@ ASSERTIONS (?<!dogs?|cats?) - causes an error at compile time. Branches that match different length - strings are permitted only at the top level of a lookbehind assertion. + causes an error at compile time. Branches that match different length + strings are permitted only at the top level of a lookbehind assertion. This is an extension compared with Perl, which requires all branches to match the same length of string. An assertion such as (?<=ab(c|de)) - is not permitted, because its single top-level branch can match two - different lengths, but it is acceptable to PCRE2 if rewritten to use + is not permitted, because its single top-level branch can match two + different lengths, but it is acceptable to PCRE2 if rewritten to use two top-level branches: (?<=abc|abde) - In some cases, the escape sequence \K (see above) can be used instead + In some cases, the escape sequence \K (see above) can be used instead of a lookbehind assertion to get round the fixed-length restriction. - The implementation of lookbehind assertions is, for each alternative, - to temporarily move the current position back by the fixed length and + The implementation of lookbehind assertions is, for each alternative, + to temporarily move the current position back by the fixed length and then try to match. If there are insufficient characters before the cur- rent position, the assertion fails. - In UTF-8 and UTF-16 modes, PCRE2 does not allow the \C escape (which - matches a single code unit even in a UTF mode) to appear in lookbehind - assertions, because it makes it impossible to calculate the length of - the lookbehind. The \X and \R escapes, which can match different num- + In UTF-8 and UTF-16 modes, PCRE2 does not allow the \C escape (which + matches a single code unit even in a UTF mode) to appear in lookbehind + assertions, because it makes it impossible to calculate the length of + the lookbehind. The \X and \R escapes, which can match different num- bers of code units, are never permitted in lookbehinds. - "Subroutine" calls (see below) such as (?2) or (?&X) are permitted in + "Subroutine" calls (see below) such as (?2) or (?&X) are permitted in lookbehinds, as long as the called capture group matches a fixed-length - string. However, recursion, that is, a "subroutine" call into a group + string. However, recursion, that is, a "subroutine" call into a group that is already active, is not supported. Perl does not support backreferences in lookbehinds. PCRE2 does support - them, but only if certain conditions are met. The - PCRE2_MATCH_UNSET_BACKREF option must not be set, there must be no use - of (?| in the pattern (it creates duplicate group numbers), and if the - backreference is by name, the name must be unique. Of course, the ref- - erenced group must itself match a fixed length substring. The following - pattern matches words containing at least two characters that begin and - end with the same character: + them, but only if certain conditions are met. The PCRE2_MATCH_UN- + SET_BACKREF option must not be set, there must be no use of (?| in the + pattern (it creates duplicate group numbers), and if the backreference + is by name, the name must be unique. Of course, the referenced group + must itself match a fixed length substring. The following pattern + matches words containing at least two characters that begin and end + with the same character: \b(\w)\w++(?<=\1) - Possessive quantifiers can be used in conjunction with lookbehind - assertions to specify efficient matching of fixed-length strings at the + Possessive quantifiers can be used in conjunction with lookbehind as- + sertions to specify efficient matching of fixed-length strings at the end of subject strings. Consider a simple pattern such as abcd$ - when applied to a long string that does not match. Because matching - proceeds from left to right, PCRE2 will look for each "a" in the sub- - ject and then see if what follows matches the rest of the pattern. If + when applied to a long string that does not match. Because matching + proceeds from left to right, PCRE2 will look for each "a" in the sub- + ject and then see if what follows matches the rest of the pattern. If the pattern is specified as ^.*abcd$ - the initial .* matches the entire string at first, but when this fails + the initial .* matches the entire string at first, but when this fails (because there is no following "a"), it backtracks to match all but the - last character, then all but the last two characters, and so on. Once - again the search for "a" covers the entire string, from right to left, + last character, then all but the last two characters, and so on. Once + again the search for "a" covers the entire string, from right to left, so we are no better off. However, if the pattern is written as ^.*+(?<=abcd) there can be no backtracking for the .*+ item because of the possessive quantifier; it can match only the entire string. The subsequent lookbe- - hind assertion does a single test on the last four characters. If it - fails, the match fails immediately. For long strings, this approach + hind assertion does a single test on the last four characters. If it + fails, the match fails immediately. For long strings, this approach makes a significant difference to the processing time. Using multiple assertions @@ -8296,18 +8435,18 @@ ASSERTIONS (?<=\d{3})(?<!999)foo - matches "foo" preceded by three digits that are not "999". Notice that - each of the assertions is applied independently at the same point in - the subject string. First there is a check that the previous three - characters are all digits, and then there is a check that the same + matches "foo" preceded by three digits that are not "999". Notice that + each of the assertions is applied independently at the same point in + the subject string. First there is a check that the previous three + characters are all digits, and then there is a check that the same three characters are not "999". This pattern does not match "foo" pre- - ceded by six characters, the first of which are digits and the last - three of which are not "999". For example, it doesn't match "123abc- + ceded by six characters, the first of which are digits and the last + three of which are not "999". For example, it doesn't match "123abc- foo". A pattern to do that is (?<=\d{3}...)(?<!999)foo - This time the first assertion looks at the preceding six characters, + This time the first assertion looks at the preceding six characters, checking that the first three are digits, and then the second assertion checks that the preceding three characters are not "999". @@ -8315,53 +8454,113 @@ ASSERTIONS (?<=(?<!foo)bar)baz - matches an occurrence of "baz" that is preceded by "bar" which in turn + matches an occurrence of "baz" that is preceded by "bar" which in turn is not preceded by "foo", while (?<=\d{3}(?!999)...)foo - is another pattern that matches "foo" preceded by three digits and any + is another pattern that matches "foo" preceded by three digits and any three characters that are not "999". +NON-ATOMIC ASSERTIONS + + The traditional Perl-compatible lookaround assertions are atomic. That + is, if an assertion is true, but there is a subsequent matching fail- + ure, there is no backtracking into the assertion. However, there are + some cases where non-atomic positive assertions can be useful. PCRE2 + provides these using the following syntax: + + (*non_atomic_positive_lookahead: or (*napla: or (?* + (*non_atomic_positive_lookbehind: or (*naplb: or (?<* + + Consider the problem of finding the right-most word in a string that + also appears earlier in the string, that is, it must appear at least + twice in total. This pattern returns the required result as captured + substring 1: + + ^(?x)(*napla: .* \b(\w++)) (?> .*? \b\1\b ){2} + + For a subject such as "word1 word2 word3 word2 word3 word4" the result + is "word3". How does it work? At the start, ^(?x) anchors the pattern + and sets the "x" option, which causes white space (introduced for read- + ability) to be ignored. Inside the assertion, the greedy .* at first + consumes the entire string, but then has to backtrack until the rest of + the assertion can match a word, which is captured by group 1. In other + words, when the assertion first succeeds, it captures the right-most + word in the string. + + The current matching point is then reset to the start of the subject, + and the rest of the pattern match checks for two occurrences of the + captured word, using an ungreedy .*? to scan from the left. If this + succeeds, we are done, but if the last word in the string does not oc- + cur twice, this part of the pattern fails. If a traditional atomic + lookhead (?= or (*pla: had been used, the assertion could not be re-en- + tered, and the whole match would fail. The pattern would succeed only + if the very last word in the subject was found twice. + + Using a non-atomic lookahead, however, means that when the last word + does not occur twice in the string, the lookahead can backtrack and + find the second-last word, and so on, until either the match succeeds, + or all words have been tested. + + Two conditions must be met for a non-atomic assertion to be useful: the + contents of one or more capturing groups must change after a backtrack + into the assertion, and there must be a backreference to a changed + group later in the pattern. If this is not the case, the rest of the + pattern match fails exactly as before because nothing has changed, so + using a non-atomic assertion just wastes resources. + + There is one exception to backtracking into a non-atomic assertion. If + an (*ACCEPT) control verb is triggered, the assertion succeeds atomi- + cally. That is, a subsequent match failure cannot backtrack into the + assertion. + + Non-atomic assertions are not supported by the alternative matching + function pcre2_dfa_match(). They are supported by JIT, but only if they + do not contain any control verbs such as (*ACCEPT). (This may change in + future). Note that assertions that appear as conditions for conditional + groups (see below) must be atomic. + + SCRIPT RUNS - In concept, a script run is a sequence of characters that are all from - the same Unicode script such as Latin or Greek. However, because some - scripts are commonly used together, and because some diacritical and - other marks are used with multiple scripts, it is not that simple. + In concept, a script run is a sequence of characters that are all from + the same Unicode script such as Latin or Greek. However, because some + scripts are commonly used together, and because some diacritical and + other marks are used with multiple scripts, it is not that simple. There is a full description of the rules that PCRE2 uses in the section entitled "Script Runs" in the pcre2unicode documentation. - If part of a pattern is enclosed between (*script_run: or (*sr: and a - closing parenthesis, it fails if the sequence of characters that it - matches are not a script run. After a failure, normal backtracking - occurs. Script runs can be used to detect spoofing attacks using char- - acters that look the same, but are from different scripts. The string - "paypal.com" is an infamous example, where the letters could be a mix- + If part of a pattern is enclosed between (*script_run: or (*sr: and a + closing parenthesis, it fails if the sequence of characters that it + matches are not a script run. After a failure, normal backtracking oc- + curs. Script runs can be used to detect spoofing attacks using charac- + ters that look the same, but are from different scripts. The string + "paypal.com" is an infamous example, where the letters could be a mix- ture of Latin and Cyrillic. This pattern ensures that the matched char- acters in a sequence of non-spaces that follow white space are a script run: \s+(*sr:\S+) - To be sure that they are all from the Latin script (for example), a + To be sure that they are all from the Latin script (for example), a lookahead can be used: \s+(?=\p{Latin})(*sr:\S+) This works as long as the first character is expected to be a character - in that script, and not (for example) punctuation, which is allowed - with any script. If this is not the case, a more creative lookahead is - needed. For example, if digits, underscore, and dots are permitted at + in that script, and not (for example) punctuation, which is allowed + with any script. If this is not the case, a more creative lookahead is + needed. For example, if digits, underscore, and dots are permitted at the start: \s+(?=[0-9_.]*\p{Latin})(*sr:\S+) - In many cases, backtracking into a script run pattern fragment is not - desirable. The script run can employ an atomic group to prevent this. - Because this is a common requirement, a shorthand notation is provided + In many cases, backtracking into a script run pattern fragment is not + desirable. The script run can employ an atomic group to prevent this. + Because this is a common requirement, a shorthand notation is provided by (*atomic_script_run: or (*asr: (*asr:...) is the same as (*sr:(?>...)) @@ -8369,13 +8568,13 @@ SCRIPT RUNS Note that the atomic group is inside the script run. Putting it outside would not prevent backtracking into the script run pattern. - Support for script runs is not available if PCRE2 is compiled without + Support for script runs is not available if PCRE2 is compiled without Unicode support. A compile-time error is given if any of the above con- - structs is encountered. Script runs are not supported by the alternate - matching function, pcre2_dfa_match() because they use the same mecha- + structs is encountered. Script runs are not supported by the alternate + matching function, pcre2_dfa_match() because they use the same mecha- nism as capturing parentheses. - Warning: The (*ACCEPT) control verb (see below) should not be used + Warning: The (*ACCEPT) control verb (see below) should not be used within a script run group, because it causes an immediate exit from the group, bypassing the script run checking. @@ -8384,116 +8583,116 @@ CONDITIONAL GROUPS It is possible to cause the matching process to obey a pattern fragment conditionally or to choose between two alternative fragments, depending - on the result of an assertion, or whether a specific capture group has + on the result of an assertion, or whether a specific capture group has already been matched. The two possible forms of conditional group are: (?(condition)yes-pattern) (?(condition)yes-pattern|no-pattern) - If the condition is satisfied, the yes-pattern is used; otherwise the - no-pattern (if present) is used. An absent no-pattern is equivalent to - an empty string (it always matches). If there are more than two alter- - natives in the group, a compile-time error occurs. Each of the two - alternatives may itself contain nested groups of any form, including - conditional groups; the restriction to two alternatives applies only at - the level of the condition itself. This pattern fragment is an example + If the condition is satisfied, the yes-pattern is used; otherwise the + no-pattern (if present) is used. An absent no-pattern is equivalent to + an empty string (it always matches). If there are more than two alter- + natives in the group, a compile-time error occurs. Each of the two al- + ternatives may itself contain nested groups of any form, including con- + ditional groups; the restriction to two alternatives applies only at + the level of the condition itself. This pattern fragment is an example where the alternatives are complex: (?(1) (A|B|C) | (D | (?(2)E|F) | E) ) There are five kinds of condition: references to capture groups, refer- - ences to recursion, two pseudo-conditions called DEFINE and VERSION, + ences to recursion, two pseudo-conditions called DEFINE and VERSION, and assertions. Checking for a used capture group by number - If the text between the parentheses consists of a sequence of digits, - the condition is true if a capture group of that number has previously - matched. If there is more than one capture group with the same number - (see the earlier section about duplicate group numbers), the condition + If the text between the parentheses consists of a sequence of digits, + the condition is true if a capture group of that number has previously + matched. If there is more than one capture group with the same number + (see the earlier section about duplicate group numbers), the condition is true if any of them have matched. An alternative notation is to pre- cede the digits with a plus or minus sign. In this case, the group num- - ber is relative rather than absolute. The most recently opened capture - group can be referenced by (?(-1), the next most recent by (?(-2), and - so on. Inside loops it can also make sense to refer to subsequent - groups. The next capture group can be referenced as (?(+1), and so on. - (The value zero in any of these forms is not used; it provokes a com- + ber is relative rather than absolute. The most recently opened capture + group can be referenced by (?(-1), the next most recent by (?(-2), and + so on. Inside loops it can also make sense to refer to subsequent + groups. The next capture group can be referenced as (?(+1), and so on. + (The value zero in any of these forms is not used; it provokes a com- pile-time error.) - Consider the following pattern, which contains non-significant white - space to make it more readable (assume the PCRE2_EXTENDED option) and + Consider the following pattern, which contains non-significant white + space to make it more readable (assume the PCRE2_EXTENDED option) and to divide it into three parts for ease of discussion: ( \( )? [^()]+ (?(1) \) ) - The first part matches an optional opening parenthesis, and if that + The first part matches an optional opening parenthesis, and if that character is present, sets it as the first captured substring. The sec- - ond part matches one or more characters that are not parentheses. The - third part is a conditional group that tests whether or not the first - capture group matched. If it did, that is, if subject started with an - opening parenthesis, the condition is true, and so the yes-pattern is - executed and a closing parenthesis is required. Otherwise, since no- + ond part matches one or more characters that are not parentheses. The + third part is a conditional group that tests whether or not the first + capture group matched. If it did, that is, if subject started with an + opening parenthesis, the condition is true, and so the yes-pattern is + executed and a closing parenthesis is required. Otherwise, since no- pattern is not present, the conditional group matches nothing. In other - words, this pattern matches a sequence of non-parentheses, optionally + words, this pattern matches a sequence of non-parentheses, optionally enclosed in parentheses. - If you were embedding this pattern in a larger one, you could use a + If you were embedding this pattern in a larger one, you could use a relative reference: ...other stuff... ( \( )? [^()]+ (?(-1) \) ) ... - This makes the fragment independent of the parentheses in the larger + This makes the fragment independent of the parentheses in the larger pattern. Checking for a used capture group by name - Perl uses the syntax (?(<name>)...) or (?('name')...) to test for a - used capture group by name. For compatibility with earlier versions of - PCRE1, which had this facility before Perl, the syntax (?(name)...) is - also recognized. Note, however, that undelimited names consisting of - the letter R followed by digits are ambiguous (see the following sec- + Perl uses the syntax (?(<name>)...) or (?('name')...) to test for a + used capture group by name. For compatibility with earlier versions of + PCRE1, which had this facility before Perl, the syntax (?(name)...) is + also recognized. Note, however, that undelimited names consisting of + the letter R followed by digits are ambiguous (see the following sec- tion). Rewriting the above example to use a named group gives this: (?<OPEN> \( )? [^()]+ (?(<OPEN>) \) ) - If the name used in a condition of this kind is a duplicate, the test - is applied to all groups of the same name, and is true if any one of + If the name used in a condition of this kind is a duplicate, the test + is applied to all groups of the same name, and is true if any one of them has matched. Checking for pattern recursion - "Recursion" in this sense refers to any subroutine-like call from one - part of the pattern to another, whether or not it is actually recur- - sive. See the sections entitled "Recursive patterns" and "Groups as + "Recursion" in this sense refers to any subroutine-like call from one + part of the pattern to another, whether or not it is actually recur- + sive. See the sections entitled "Recursive patterns" and "Groups as subroutines" below for details of recursion and subroutine calls. - If a condition is the string (R), and there is no capture group with - the name R, the condition is true if matching is currently in a recur- - sion or subroutine call to the whole pattern or any capture group. If - digits follow the letter R, and there is no group with that name, the - condition is true if the most recent call is into a group with the - given number, which must exist somewhere in the overall pattern. This + If a condition is the string (R), and there is no capture group with + the name R, the condition is true if matching is currently in a recur- + sion or subroutine call to the whole pattern or any capture group. If + digits follow the letter R, and there is no group with that name, the + condition is true if the most recent call is into a group with the + given number, which must exist somewhere in the overall pattern. This is a contrived example that is equivalent to a+b: ((?(R1)a+|(?1)b)) - However, in both cases, if there is a capture group with a matching - name, the condition tests for its being set, as described in the sec- - tion above, instead of testing for recursion. For example, creating a - group with the name R1 by adding (?<R1>) to the above pattern com- + However, in both cases, if there is a capture group with a matching + name, the condition tests for its being set, as described in the sec- + tion above, instead of testing for recursion. For example, creating a + group with the name R1 by adding (?<R1>) to the above pattern com- pletely changes its meaning. If a name preceded by ampersand follows the letter R, for example: (?(R&name)...) - the condition is true if the most recent recursion is into a group of + the condition is true if the most recent recursion is into a group of that name (which must exist within the pattern). This condition does not check the entire recursion stack. It tests only - the current level. If the name used in a condition of this kind is a - duplicate, the test is applied to all groups of the same name, and is + the current level. If the name used in a condition of this kind is a + duplicate, the test is applied to all groups of the same name, and is true if any one of them is the most recent recursion. At "top level", all these recursion test conditions are false. @@ -8501,14 +8700,13 @@ CONDITIONAL GROUPS Defining capture groups for use by reference only If the condition is the string (DEFINE), the condition is always false, - even if there is a group with the name DEFINE. In this case, there may - be only one alternative in the rest of the conditional group. It is - always skipped if control reaches this point in the pattern; the idea - of DEFINE is that it can be used to define subroutines that can be ref- - erenced from elsewhere. (The use of subroutines is described below.) - For example, a pattern to match an IPv4 address such as - "192.168.23.245" could be written like this (ignore white space and - line breaks): + even if there is a group with the name DEFINE. In this case, there may + be only one alternative in the rest of the conditional group. It is al- + ways skipped if control reaches this point in the pattern; the idea of + DEFINE is that it can be used to define subroutines that can be refer- + enced from elsewhere. (The use of subroutines is described below.) For + example, a pattern to match an IPv4 address such as "192.168.23.245" + could be written like this (ignore white space and line breaks): (?(DEFINE) (?<byte> 2[0-4]\d | 25[0-5] | 1\d\d | [1-9]?\d) ) \b (?&byte) (\.(?&byte)){3} \b @@ -8541,17 +8739,19 @@ CONDITIONAL GROUPS If the condition is not in any of the above formats, it must be a parenthesized assertion. This may be a positive or negative lookahead - or lookbehind assertion. Consider this pattern, again containing non- - significant white space, and with the two alternatives on the second - line: + or lookbehind assertion. However, it must be a traditional atomic as- + sertion, not one of the PCRE2-specific non-atomic assertions. + + Consider this pattern, again containing non-significant white space, + and with the two alternatives on the second line: (?(?=[^a-z]*[a-z]) \d{2}-[a-z]{3}-\d{2} | \d{2}-\d{2}-\d{2} ) - The condition is a positive lookahead assertion that matches an - optional sequence of non-letters followed by a letter. In other words, - it tests for the presence of at least one letter in the subject. If a - letter is found, the subject is matched against the first alternative; + The condition is a positive lookahead assertion that matches an op- + tional sequence of non-letters followed by a letter. In other words, it + tests for the presence of at least one letter in the subject. If a let- + ter is found, the subject is matched against the first alternative; otherwise it is matched against the second. This pattern matches strings in one of the two forms dd-aaa-dd or dd-dd-dd, where aaa are letters and dd are digits. @@ -8613,18 +8813,18 @@ RECURSIVE PATTERNS The (?p{...}) item interpolates Perl code at run time, and in this case refers recursively to the pattern in which it appears. - Obviously, PCRE2 cannot support the interpolation of Perl code. - Instead, it supports special syntax for recursion of the entire pat- - tern, and also for individual capture group recursion. After its intro- - duction in PCRE1 and Python, this kind of recursion was subsequently - introduced into Perl at release 5.10. + Obviously, PCRE2 cannot support the interpolation of Perl code. In- + stead, it supports special syntax for recursion of the entire pattern, + and also for individual capture group recursion. After its introduction + in PCRE1 and Python, this kind of recursion was subsequently introduced + into Perl at release 5.10. A special item that consists of (? followed by a number greater than zero and a closing parenthesis is a recursive subroutine call of the capture group of the given number, provided that it occurs inside that - group. (If not, it is a non-recursive subroutine call, which is - described in the next section.) The special item (?R) or (?0) is a - recursive call of the entire regular expression. + group. (If not, it is a non-recursive subroutine call, which is de- + scribed in the next section.) The special item (?R) or (?0) is a recur- + sive call of the entire regular expression. This PCRE2 pattern solves the nested parentheses problem (assume the PCRE2_EXTENDED option is set so that white space is ignored): @@ -8632,10 +8832,10 @@ RECURSIVE PATTERNS \( ( [^()]++ | (?R) )* \) First it matches an opening parenthesis. Then it matches any number of - substrings which can either be a sequence of non-parentheses, or a - recursive match of the pattern itself (that is, a correctly parenthe- - sized substring). Finally there is a closing parenthesis. Note the use - of a possessive quantifier to avoid backtracking into sequences of non- + substrings which can either be a sequence of non-parentheses, or a re- + cursive match of the pattern itself (that is, a correctly parenthesized + substring). Finally there is a closing parenthesis. Note the use of a + possessive quantifier to avoid backtracking into sequences of non- parentheses. If this were part of a larger pattern, you would not want to recurse @@ -8735,54 +8935,53 @@ RECURSIVE PATTERNS now compatible with the way Perl works. If you want a subroutine call to be atomic, you must explicitly enclose it in an atomic group. - Supporting backtracking into recursions simplifies certain types of - recursive pattern. For example, this pattern matches palindromic - strings: + Supporting backtracking into recursions simplifies certain types of re- + cursive pattern. For example, this pattern matches palindromic strings: ^((.)(?1)\2|.?)$ - The second branch in the group matches a single central character in - the palindrome when there are an odd number of characters, or nothing - when there are an even number of characters, but in order to work it - has to be able to try the second case when the rest of the pattern + The second branch in the group matches a single central character in + the palindrome when there are an odd number of characters, or nothing + when there are an even number of characters, but in order to work it + has to be able to try the second case when the rest of the pattern match fails. If you want to match typical palindromic phrases, the pat- - tern has to ignore all non-word characters, which can be done like + tern has to ignore all non-word characters, which can be done like this: ^\W*+((.)\W*+(?1)\W*+\2|\W*+.?)\W*+$ - If run with the PCRE2_CASELESS option, this pattern matches phrases - such as "A man, a plan, a canal: Panama!". Note the use of the posses- - sive quantifier *+ to avoid backtracking into sequences of non-word + If run with the PCRE2_CASELESS option, this pattern matches phrases + such as "A man, a plan, a canal: Panama!". Note the use of the posses- + sive quantifier *+ to avoid backtracking into sequences of non-word characters. Without this, PCRE2 takes a great deal longer (ten times or - more) to match typical phrases, and Perl takes so long that you think + more) to match typical phrases, and Perl takes so long that you think it has gone into a loop. - Another way in which PCRE2 and Perl used to differ in their recursion - processing is in the handling of captured values. Formerly in Perl, - when a group was called recursively or as a subroutine (see the next + Another way in which PCRE2 and Perl used to differ in their recursion + processing is in the handling of captured values. Formerly in Perl, + when a group was called recursively or as a subroutine (see the next section), it had no access to any values that were captured outside the - recursion, whereas in PCRE2 these values can be referenced. Consider + recursion, whereas in PCRE2 these values can be referenced. Consider this pattern: ^(.)(\1|a(?2)) - This pattern matches "bab". The first capturing parentheses match "b", + This pattern matches "bab". The first capturing parentheses match "b", then in the second group, when the backreference \1 fails to match "b", the second alternative matches "a" and then recurses. In the recursion, - \1 does now match "b" and so the whole match succeeds. This match used + \1 does now match "b" and so the whole match succeeds. This match used to fail in Perl, but in later versions (I tried 5.024) it now works. GROUPS AS SUBROUTINES - If the syntax for a recursive group call (either by number or by name) - is used outside the parentheses to which it refers, it operates a bit - like a subroutine in a programming language. More accurately, PCRE2 + If the syntax for a recursive group call (either by number or by name) + is used outside the parentheses to which it refers, it operates a bit + like a subroutine in a programming language. More accurately, PCRE2 treats the referenced group as an independent subpattern which it tries - to match at the current matching position. The called group may be - defined before or after the reference. A numbered reference can be - absolute or relative, as in these examples: + to match at the current matching position. The called group may be de- + fined before or after the reference. A numbered reference can be abso- + lute or relative, as in these examples: (...(absolute)...)...(?2)... (...(relative)...)...(?-1)... @@ -8792,106 +8991,106 @@ GROUPS AS SUBROUTINES (sens|respons)e and \1ibility - matches "sense and sensibility" and "response and responsibility", but + matches "sense and sensibility" and "response and responsibility", but not "sense and responsibility". If instead the pattern (sens|respons)e and (?1)ibility - is used, it does match "sense and responsibility" as well as the other - two strings. Another example is given in the discussion of DEFINE + is used, it does match "sense and responsibility" as well as the other + two strings. Another example is given in the discussion of DEFINE above. - Like recursions, subroutine calls used to be treated as atomic, but - this changed at PCRE2 release 10.30, so backtracking into subroutine - calls can now occur. However, any capturing parentheses that are set + Like recursions, subroutine calls used to be treated as atomic, but + this changed at PCRE2 release 10.30, so backtracking into subroutine + calls can now occur. However, any capturing parentheses that are set during the subroutine call revert to their previous values afterwards. - Processing options such as case-independence are fixed when a group is - defined, so if it is used as a subroutine, such options cannot be + Processing options such as case-independence are fixed when a group is + defined, so if it is used as a subroutine, such options cannot be changed for different calls. For example, consider this pattern: (abc)(?i:(?-1)) - It matches "abcabc". It does not match "abcABC" because the change of + It matches "abcabc". It does not match "abcABC" because the change of processing option does not affect the called group. - The behaviour of backtracking control verbs in groups when called as + The behaviour of backtracking control verbs in groups when called as subroutines is described in the section entitled "Backtracking verbs in subroutines" below. ONIGURUMA SUBROUTINE SYNTAX - For compatibility with Oniguruma, the non-Perl syntax \g followed by a + For compatibility with Oniguruma, the non-Perl syntax \g followed by a name or a number enclosed either in angle brackets or single quotes, is - an alternative syntax for calling a group as a subroutine, possibly - recursively. Here are two of the examples used above, rewritten using + an alternative syntax for calling a group as a subroutine, possibly re- + cursively. Here are two of the examples used above, rewritten using this syntax: (?<pn> \( ( (?>[^()]+) | \g<pn> )* \) ) (sens|respons)e and \g'1'ibility - PCRE2 supports an extension to Oniguruma: if a number is preceded by a + PCRE2 supports an extension to Oniguruma: if a number is preceded by a plus or a minus sign it is taken as a relative reference. For example: (abc)(?i:\g<-1>) - Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are not - synonymous. The former is a backreference; the latter is a subroutine + Note that \g{...} (Perl syntax) and \g<...> (Oniguruma syntax) are not + synonymous. The former is a backreference; the latter is a subroutine call. CALLOUTS Perl has a feature whereby using the sequence (?{...}) causes arbitrary - Perl code to be obeyed in the middle of matching a regular expression. + Perl code to be obeyed in the middle of matching a regular expression. This makes it possible, amongst other things, to extract different sub- strings that match the same pair of parentheses when there is a repeti- tion. - PCRE2 provides a similar feature, but of course it cannot obey arbi- - trary Perl code. The feature is called "callout". The caller of PCRE2 - provides an external function by putting its entry point in a match - context using the function pcre2_set_callout(), and then passing that - context to pcre2_match() or pcre2_dfa_match(). If no match context is + PCRE2 provides a similar feature, but of course it cannot obey arbi- + trary Perl code. The feature is called "callout". The caller of PCRE2 + provides an external function by putting its entry point in a match + context using the function pcre2_set_callout(), and then passing that + context to pcre2_match() or pcre2_dfa_match(). If no match context is passed, or if the callout entry point is set to NULL, callouts are dis- abled. - Within a regular expression, (?C<arg>) indicates a point at which the - external function is to be called. There are two kinds of callout: - those with a numerical argument and those with a string argument. (?C) - on its own with no argument is treated as (?C0). A numerical argument - allows the application to distinguish between different callouts. - String arguments were added for release 10.20 to make it possible for - script languages that use PCRE2 to embed short scripts within patterns + Within a regular expression, (?C<arg>) indicates a point at which the + external function is to be called. There are two kinds of callout: + those with a numerical argument and those with a string argument. (?C) + on its own with no argument is treated as (?C0). A numerical argument + allows the application to distinguish between different callouts. + String arguments were added for release 10.20 to make it possible for + script languages that use PCRE2 to embed short scripts within patterns in a similar way to Perl. During matching, when PCRE2 reaches a callout point, the external func- - tion is called. It is provided with the number or string argument of - the callout, the position in the pattern, and one item of data that is + tion is called. It is provided with the number or string argument of + the callout, the position in the pattern, and one item of data that is also set in the match block. The callout function may cause matching to proceed, to backtrack, or to fail. - By default, PCRE2 implements a number of optimizations at matching - time, and one side-effect is that sometimes callouts are skipped. If - you need all possible callouts to happen, you need to set options that - disable the relevant optimizations. More details, including a complete - description of the programming interface to the callout function, are + By default, PCRE2 implements a number of optimizations at matching + time, and one side-effect is that sometimes callouts are skipped. If + you need all possible callouts to happen, you need to set options that + disable the relevant optimizations. More details, including a complete + description of the programming interface to the callout function, are given in the pcre2callout documentation. Callouts with numerical arguments - If you just want to have a means of identifying different callout - points, put a number less than 256 after the letter C. For example, + If you just want to have a means of identifying different callout + points, put a number less than 256 after the letter C. For example, this pattern has two callout points: (?C1)abc(?C2)def - If the PCRE2_AUTO_CALLOUT flag is passed to pcre2_compile(), numerical - callouts are automatically installed before each item in the pattern. - They are all numbered 255. If there is a conditional group in the pat- + If the PCRE2_AUTO_CALLOUT flag is passed to pcre2_compile(), numerical + callouts are automatically installed before each item in the pattern. + They are all numbered 255. If there is a conditional group in the pat- tern whose condition is an assertion, an additional callout is inserted - just before the condition. An explicit callout may also be set at this + just before the condition. An explicit callout may also be set at this position, as in this example: (?(?C9)(?=a)abc|def) @@ -8901,78 +9100,78 @@ CALLOUTS Callouts with string arguments - A delimited string may be used instead of a number as a callout argu- - ment. The starting delimiter must be one of ` ' " ^ % # $ { and the + A delimited string may be used instead of a number as a callout argu- + ment. The starting delimiter must be one of ` ' " ^ % # $ { and the ending delimiter is the same as the start, except for {, where the end- - ing delimiter is }. If the ending delimiter is needed within the + ing delimiter is }. If the ending delimiter is needed within the string, it must be doubled. For example: (?C'ab ''c'' d')xyz(?C{any text})pqr - The doubling is removed before the string is passed to the callout + The doubling is removed before the string is passed to the callout function. BACKTRACKING CONTROL - There are a number of special "Backtracking Control Verbs" (to use - Perl's terminology) that modify the behaviour of backtracking during - matching. They are generally of the form (*VERB) or (*VERB:NAME). Some - verbs take either form, possibly behaving differently depending on - whether or not a name is present. The names are not required to be + There are a number of special "Backtracking Control Verbs" (to use + Perl's terminology) that modify the behaviour of backtracking during + matching. They are generally of the form (*VERB) or (*VERB:NAME). Some + verbs take either form, and may behave differently depending on whether + or not a name argument is present. The names are not required to be unique within the pattern. - By default, for compatibility with Perl, a name is any sequence of + By default, for compatibility with Perl, a name is any sequence of characters that does not include a closing parenthesis. The name is not - processed in any way, and it is not possible to include a closing - parenthesis in the name. This can be changed by setting the - PCRE2_ALT_VERBNAMES option, but the result is no longer Perl-compati- + processed in any way, and it is not possible to include a closing + parenthesis in the name. This can be changed by setting the + PCRE2_ALT_VERBNAMES option, but the result is no longer Perl-compati- ble. - When PCRE2_ALT_VERBNAMES is set, backslash processing is applied to - verb names and only an unescaped closing parenthesis terminates the - name. However, the only backslash items that are permitted are \Q, \E, - and sequences such as \x{100} that define character code points. Char- + When PCRE2_ALT_VERBNAMES is set, backslash processing is applied to + verb names and only an unescaped closing parenthesis terminates the + name. However, the only backslash items that are permitted are \Q, \E, + and sequences such as \x{100} that define character code points. Char- acter type escapes such as \d are faulted. A closing parenthesis can be included in a name either as \) or between - \Q and \E. In addition to backslash processing, if the PCRE2_EXTENDED + \Q and \E. In addition to backslash processing, if the PCRE2_EXTENDED or PCRE2_EXTENDED_MORE option is also set, unescaped whitespace in verb names is skipped, and #-comments are recognized, exactly as in the rest - of the pattern. PCRE2_EXTENDED and PCRE2_EXTENDED_MORE do not affect + of the pattern. PCRE2_EXTENDED and PCRE2_EXTENDED_MORE do not affect verb names unless PCRE2_ALT_VERBNAMES is also set. - The maximum length of a name is 255 in the 8-bit library and 65535 in - the 16-bit and 32-bit libraries. If the name is empty, that is, if the - closing parenthesis immediately follows the colon, the effect is as if + The maximum length of a name is 255 in the 8-bit library and 65535 in + the 16-bit and 32-bit libraries. If the name is empty, that is, if the + closing parenthesis immediately follows the colon, the effect is as if the colon were not there. Any number of these verbs may occur in a pat- - tern. + tern. Except for (*ACCEPT), they may not be quantified. - Since these verbs are specifically related to backtracking, most of - them can be used only when the pattern is to be matched using the tra- + Since these verbs are specifically related to backtracking, most of + them can be used only when the pattern is to be matched using the tra- ditional matching function, because that uses a backtracking algorithm. - With the exception of (*FAIL), which behaves like a failing negative + With the exception of (*FAIL), which behaves like a failing negative assertion, the backtracking control verbs cause an error if encountered by the DFA matching function. - The behaviour of these verbs in repeated groups, assertions, and in - capture groups called as subroutines (whether or not recursively) is + The behaviour of these verbs in repeated groups, assertions, and in + capture groups called as subroutines (whether or not recursively) is documented below. Optimizations that affect backtracking verbs PCRE2 contains some optimizations that are used to speed up matching by running some checks at the start of each match attempt. For example, it - may know the minimum length of matching subject, or that a particular + may know the minimum length of matching subject, or that a particular character must be present. When one of these optimizations bypasses the - running of a match, any included backtracking verbs will not, of + running of a match, any included backtracking verbs will not, of course, be processed. You can suppress the start-of-match optimizations - by setting the PCRE2_NO_START_OPTIMIZE option when calling pcre2_com- - pile(), or by starting the pattern with (*NO_START_OPT). There is more + by setting the PCRE2_NO_START_OPTIMIZE option when calling pcre2_com- + pile(), or by starting the pattern with (*NO_START_OPT). There is more discussion of this option in the section entitled "Compiling a pattern" in the pcre2api documentation. - Experiments with Perl suggest that it too has similar optimizations, + Experiments with Perl suggest that it too has similar optimizations, and like PCRE2, turning them off can change the result of a match. Verbs that act immediately @@ -8981,47 +9180,60 @@ BACKTRACKING CONTROL (*ACCEPT) or (*ACCEPT:NAME) - This verb causes the match to end successfully, skipping the remainder - of the pattern. However, when it is inside a capture group that is + This verb causes the match to end successfully, skipping the remainder + of the pattern. However, when it is inside a capture group that is called as a subroutine, only that group is ended successfully. Matching then continues at the outer level. If (*ACCEPT) in triggered in a posi- - tive assertion, the assertion succeeds; in a negative assertion, the + tive assertion, the assertion succeeds; in a negative assertion, the assertion fails. - If (*ACCEPT) is inside capturing parentheses, the data so far is cap- + If (*ACCEPT) is inside capturing parentheses, the data so far is cap- tured. For example: A((?:A|B(*ACCEPT)|C)D) - This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is cap- + This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is cap- tured by the outer parentheses. - Warning: (*ACCEPT) should not be used within a script run group, - because it causes an immediate exit from the group, bypassing the - script run checking. + (*ACCEPT) is the only backtracking verb that is allowed to be quanti- + fied because an ungreedy quantification with a minimum of zero acts + only when a backtrack happens. Consider, for example, + + (A(*ACCEPT)??B)C + + where A, B, and C may be complex expressions. After matching "A", the + matcher processes "BC"; if that fails, causing a backtrack, (*ACCEPT) + is triggered and the match succeeds. In both cases, all but C is cap- + tured. Whereas (*COMMIT) (see below) means "fail on backtrack", a re- + peated (*ACCEPT) of this type means "succeed on backtrack". + + Warning: (*ACCEPT) should not be used within a script run group, be- + cause it causes an immediate exit from the group, bypassing the script + run checking. (*FAIL) or (*FAIL:NAME) - This verb causes a matching failure, forcing backtracking to occur. It - may be abbreviated to (*F). It is equivalent to (?!) but easier to + This verb causes a matching failure, forcing backtracking to occur. It + may be abbreviated to (*F). It is equivalent to (?!) but easier to read. The Perl documentation notes that it is probably useful only when combined with (?{}) or (??{}). Those are, of course, Perl features that - are not present in PCRE2. The nearest equivalent is the callout fea- + are not present in PCRE2. The nearest equivalent is the callout fea- ture, as for example in this pattern: a+(?C)(*FAIL) - A match with the string "aaaa" always fails, but the callout is taken + A match with the string "aaaa" always fails, but the callout is taken before each backtrack happens (in this example, 10 times). - (*ACCEPT:NAME) and (*FAIL:NAME) are treated as (*MARK:NAME)(*ACCEPT) - and (*MARK:NAME)(*FAIL), respectively. + (*ACCEPT:NAME) and (*FAIL:NAME) behave the same as (*MARK:NAME)(*AC- + CEPT) and (*MARK:NAME)(*FAIL), respectively, that is, a (*MARK) is + recorded just before the verb acts. Recording which path was taken - There is one verb whose main purpose is to track how a match was - arrived at, though it also has a secondary use in conjunction with - advancing the match starting point (see (*SKIP) below). + There is one verb whose main purpose is to track how a match was ar- + rived at, though it also has a secondary use in conjunction with ad- + vancing the match starting point (see (*SKIP) below). (*MARK:NAME) or (*:NAME) @@ -9118,8 +9330,8 @@ BACKTRACKING CONTROL (*COMMIT) during a match does not always guarantee that a match must be at this starting point. - Note that (*COMMIT) at the start of a pattern is not the same as an - anchor, unless PCRE2's start-of-match optimizations are turned off, as + Note that (*COMMIT) at the start of a pattern is not the same as an an- + chor, unless PCRE2's start-of-match optimizations are turned off, as shown in this output from pcre2test: re> /(*COMMIT)abc/ @@ -9172,21 +9384,26 @@ BACKTRACKING CONTROL (starting at the first character in the string), the starting point skips on to start the next attempt at "c". Note that a possessive quan- tifer does not have the same effect as this example; although it would - suppress backtracking during the first match attempt, the second - attempt would start at the second character instead of skipping on to + suppress backtracking during the first match attempt, the second at- + tempt would start at the second character instead of skipping on to "c". + If (*SKIP) is used to specify a new starting position that is the same + as the starting position of the current match, or (by being inside a + lookbehind) earlier, the position specified by (*SKIP) is ignored, and + instead the normal "bumpalong" occurs. + (*SKIP:NAME) - When (*SKIP) has an associated name, its behaviour is modified. When - such a (*SKIP) is triggered, the previous path through the pattern is - searched for the most recent (*MARK) that has the same name. If one is - found, the "bumpalong" advance is to the subject position that corre- - sponds to that (*MARK) instead of to where (*SKIP) was encountered. If + When (*SKIP) has an associated name, its behaviour is modified. When + such a (*SKIP) is triggered, the previous path through the pattern is + searched for the most recent (*MARK) that has the same name. If one is + found, the "bumpalong" advance is to the subject position that corre- + sponds to that (*MARK) instead of to where (*SKIP) was encountered. If no (*MARK) with a matching name is found, the (*SKIP) is ignored. - The search for a (*MARK) name uses the normal backtracking mechanism, - which means that it does not see (*MARK) settings that are inside + The search for a (*MARK) name uses the normal backtracking mechanism, + which means that it does not see (*MARK) settings that are inside atomic groups or assertions, because they are never re-entered by back- tracking. Compare the following pcre2test examples: @@ -9200,105 +9417,105 @@ BACKTRACKING CONTROL 0: b 1: b - In the first example, the (*MARK) setting is in an atomic group, so it + In the first example, the (*MARK) setting is in an atomic group, so it is not seen when (*SKIP:X) triggers, causing the (*SKIP) to be ignored. - This allows the second branch of the pattern to be tried at the first - character position. In the second example, the (*MARK) setting is not - in an atomic group. This allows (*SKIP:X) to find the (*MARK) when it + This allows the second branch of the pattern to be tried at the first + character position. In the second example, the (*MARK) setting is not + in an atomic group. This allows (*SKIP:X) to find the (*MARK) when it backtracks, and this causes a new matching attempt to start at the sec- - ond character. This time, the (*MARK) is never seen because "a" does + ond character. This time, the (*MARK) is never seen because "a" does not match "b", so the matcher immediately jumps to the second branch of the pattern. - Note that (*SKIP:NAME) searches only for names set by (*MARK:NAME). It + Note that (*SKIP:NAME) searches only for names set by (*MARK:NAME). It ignores names that are set by other backtracking verbs. (*THEN) or (*THEN:NAME) - This verb causes a skip to the next innermost alternative when back- - tracking reaches it. That is, it cancels any further backtracking - within the current alternative. Its name comes from the observation + This verb causes a skip to the next innermost alternative when back- + tracking reaches it. That is, it cancels any further backtracking + within the current alternative. Its name comes from the observation that it can be used for a pattern-based if-then-else block: ( COND1 (*THEN) FOO | COND2 (*THEN) BAR | COND3 (*THEN) BAZ ) ... - If the COND1 pattern matches, FOO is tried (and possibly further items - after the end of the group if FOO succeeds); on failure, the matcher - skips to the second alternative and tries COND2, without backtracking - into COND1. If that succeeds and BAR fails, COND3 is tried. If subse- - quently BAZ fails, there are no more alternatives, so there is a back- - track to whatever came before the entire group. If (*THEN) is not - inside an alternation, it acts like (*PRUNE). + If the COND1 pattern matches, FOO is tried (and possibly further items + after the end of the group if FOO succeeds); on failure, the matcher + skips to the second alternative and tries COND2, without backtracking + into COND1. If that succeeds and BAR fails, COND3 is tried. If subse- + quently BAZ fails, there are no more alternatives, so there is a back- + track to whatever came before the entire group. If (*THEN) is not in- + side an alternation, it acts like (*PRUNE). - The behaviour of (*THEN:NAME) is not the same as (*MARK:NAME)(*THEN). + The behaviour of (*THEN:NAME) is not the same as (*MARK:NAME)(*THEN). It is like (*MARK:NAME) in that the name is remembered for passing back - to the caller. However, (*SKIP:NAME) searches only for names set with + to the caller. However, (*SKIP:NAME) searches only for names set with (*MARK), ignoring those set by other backtracking verbs. - A group that does not contain a | character is just a part of the - enclosing alternative; it is not a nested alternation with only one - alternative. The effect of (*THEN) extends beyond such a group to the - enclosing alternative. Consider this pattern, where A, B, etc. are - complex pattern fragments that do not contain any | characters at this + A group that does not contain a | character is just a part of the en- + closing alternative; it is not a nested alternation with only one al- + ternative. The effect of (*THEN) extends beyond such a group to the en- + closing alternative. Consider this pattern, where A, B, etc. are com- + plex pattern fragments that do not contain any | characters at this level: A (B(*THEN)C) | D - If A and B are matched, but there is a failure in C, matching does not + If A and B are matched, but there is a failure in C, matching does not backtrack into A; instead it moves to the next alternative, that is, D. - However, if the group containing (*THEN) is given an alternative, it + However, if the group containing (*THEN) is given an alternative, it behaves differently: A (B(*THEN)C | (*FAIL)) | D The effect of (*THEN) is now confined to the inner group. After a fail- - ure in C, matching moves to (*FAIL), which causes the whole group to - fail because there are no more alternatives to try. In this case, + ure in C, matching moves to (*FAIL), which causes the whole group to + fail because there are no more alternatives to try. In this case, matching does backtrack into A. - Note that a conditional group is not considered as having two alterna- - tives, because only one is ever used. In other words, the | character - in a conditional group has a different meaning. Ignoring white space, + Note that a conditional group is not considered as having two alterna- + tives, because only one is ever used. In other words, the | character + in a conditional group has a different meaning. Ignoring white space, consider: ^.*? (?(?=a) a | b(*THEN)c ) - If the subject is "ba", this pattern does not match. Because .*? is - ungreedy, it initially matches zero characters. The condition (?=a) - then fails, the character "b" is matched, but "c" is not. At this - point, matching does not backtrack to .*? as might perhaps be expected - from the presence of the | character. The conditional group is part of - the single alternative that comprises the whole pattern, and so the - match fails. (If there was a backtrack into .*?, allowing it to match - "b", the match would succeed.) + If the subject is "ba", this pattern does not match. Because .*? is un- + greedy, it initially matches zero characters. The condition (?=a) then + fails, the character "b" is matched, but "c" is not. At this point, + matching does not backtrack to .*? as might perhaps be expected from + the presence of the | character. The conditional group is part of the + single alternative that comprises the whole pattern, and so the match + fails. (If there was a backtrack into .*?, allowing it to match "b", + the match would succeed.) - The verbs just described provide four different "strengths" of control + The verbs just described provide four different "strengths" of control when subsequent matching fails. (*THEN) is the weakest, carrying on the - match at the next alternative. (*PRUNE) comes next, failing the match - at the current starting position, but allowing an advance to the next - character (for an unanchored pattern). (*SKIP) is similar, except that + match at the next alternative. (*PRUNE) comes next, failing the match + at the current starting position, but allowing an advance to the next + character (for an unanchored pattern). (*SKIP) is similar, except that the advance may be more than one character. (*COMMIT) is the strongest, causing the entire match to fail. More than one backtracking verb - If more than one backtracking verb is present in a pattern, the one - that is backtracked onto first acts. For example, consider this pat- + If more than one backtracking verb is present in a pattern, the one + that is backtracked onto first acts. For example, consider this pat- tern, where A, B, etc. are complex pattern fragments: (A(*COMMIT)B(*THEN)C|ABD) - If A matches but B fails, the backtrack to (*COMMIT) causes the entire + If A matches but B fails, the backtrack to (*COMMIT) causes the entire match to fail. However, if A and B match, but C fails, the backtrack to - (*THEN) causes the next alternative (ABD) to be tried. This behaviour - is consistent, but is not always the same as Perl's. It means that if - two or more backtracking verbs appear in succession, all the the last + (*THEN) causes the next alternative (ABD) to be tried. This behaviour + is consistent, but is not always the same as Perl's. It means that if + two or more backtracking verbs appear in succession, all the the last of them has no effect. Consider this example: ...(*COMMIT)(*PRUNE)... If there is a matching failure to the right, backtracking onto (*PRUNE) - causes it to be triggered, and its action is taken. There can never be + causes it to be triggered, and its action is taken. There can never be a backtrack onto (*COMMIT). Backtracking verbs in repeated groups @@ -9308,42 +9525,50 @@ BACKTRACKING CONTROL /(a(*COMMIT)b)+ac/ - If the subject is "abac", Perl matches unless its optimizations are - disabled, but PCRE2 always fails because the (*COMMIT) in the second + If the subject is "abac", Perl matches unless its optimizations are + disabled, but PCRE2 always fails because the (*COMMIT) in the second repeat of the group acts. Backtracking verbs in assertions - (*FAIL) in any assertion has its normal effect: it forces an immediate - backtrack. The behaviour of the other backtracking verbs depends on - whether or not the assertion is standalone or acting as the condition + (*FAIL) in any assertion has its normal effect: it forces an immediate + backtrack. The behaviour of the other backtracking verbs depends on + whether or not the assertion is standalone or acting as the condition in a conditional group. - (*ACCEPT) in a standalone positive assertion causes the assertion to - succeed without any further processing; captured strings and a mark - name (if set) are retained. In a standalone negative assertion, - (*ACCEPT) causes the assertion to fail without any further processing; - captured substrings and any mark name are discarded. + (*ACCEPT) in a standalone positive assertion causes the assertion to + succeed without any further processing; captured strings and a mark + name (if set) are retained. In a standalone negative assertion, (*AC- + CEPT) causes the assertion to fail without any further processing; cap- + tured substrings and any mark name are discarded. - If the assertion is a condition, (*ACCEPT) causes the condition to be - true for a positive assertion and false for a negative one; captured + If the assertion is a condition, (*ACCEPT) causes the condition to be + true for a positive assertion and false for a negative one; captured substrings are retained in both cases. The remaining verbs act only when a later failure causes a backtrack to - reach them. This means that their effect is confined to the assertion, - because lookaround assertions are atomic. A backtrack that occurs after - an assertion is complete does not jump back into the assertion. Note in - particular that a (*MARK) name that is set in an assertion is not - "seen" by an instance of (*SKIP:NAME) latter in the pattern. - - The effect of (*THEN) is not allowed to escape beyond an assertion. If - there are no more branches to try, (*THEN) causes a positive assertion + reach them. This means that, for the Perl-compatible assertions, their + effect is confined to the assertion, because Perl lookaround assertions + are atomic. A backtrack that occurs after such an assertion is complete + does not jump back into the assertion. Note in particular that a + (*MARK) name that is set in an assertion is not "seen" by an instance + of (*SKIP:NAME) later in the pattern. + + PCRE2 now supports non-atomic positive assertions, as described in the + section entitled "Non-atomic assertions" above. These assertions must + be standalone (not used as conditions). They are not Perl-compatible. + For these assertions, a later backtrack does jump back into the asser- + tion, and therefore verbs such as (*COMMIT) can be triggered by back- + tracks from later in the pattern. + + The effect of (*THEN) is not allowed to escape beyond an assertion. If + there are no more branches to try, (*THEN) causes a positive assertion to be false, and a negative assertion to be true. - The other backtracking verbs are not treated specially if they appear - in a standalone positive assertion. In a conditional positive asser- + The other backtracking verbs are not treated specially if they appear + in a standalone positive assertion. In a conditional positive asser- tion, backtracking (from within the assertion) into (*COMMIT), (*SKIP), - or (*PRUNE) causes the condition to be false. However, for both stand- + or (*PRUNE) causes the condition to be false. However, for both stand- alone and conditional negative assertions, backtracking into (*COMMIT), (*SKIP), or (*PRUNE) causes the assertion to be true, without consider- ing any further alternative branches. @@ -9353,26 +9578,26 @@ BACKTRACKING CONTROL These behaviours occur whether or not the group is called recursively. (*ACCEPT) in a group called as a subroutine causes the subroutine match - to succeed without any further processing. Matching then continues - after the subroutine call. Perl documents this behaviour. Perl's treat- + to succeed without any further processing. Matching then continues af- + ter the subroutine call. Perl documents this behaviour. Perl's treat- ment of the other verbs in subroutines is different in some cases. - (*FAIL) in a group called as a subroutine has its normal effect: it + (*FAIL) in a group called as a subroutine has its normal effect: it forces an immediate backtrack. - (*COMMIT), (*SKIP), and (*PRUNE) cause the subroutine match to fail - when triggered by being backtracked to in a group called as a subrou- + (*COMMIT), (*SKIP), and (*PRUNE) cause the subroutine match to fail + when triggered by being backtracked to in a group called as a subrou- tine. There is then a backtrack at the outer level. (*THEN), when triggered, skips to the next alternative in the innermost - enclosing group that has alternatives (its normal behaviour). However, + enclosing group that has alternatives (its normal behaviour). However, if there is no such group within the subroutine's group, the subroutine match fails and there is a backtrack at the outer level. SEE ALSO - pcre2api(3), pcre2callout(3), pcre2matching(3), pcre2syntax(3), + pcre2api(3), pcre2callout(3), pcre2matching(3), pcre2syntax(3), pcre2(3). @@ -9385,8 +9610,8 @@ AUTHOR REVISION - Last updated: 12 February 2019 - Copyright (c) 1997-2019 University of Cambridge. + Last updated: 06 October 2020 + Copyright (c) 1997-2020 University of Cambridge. ------------------------------------------------------------------------------ @@ -9469,9 +9694,9 @@ STACK AND HEAP USAGE AT RUN TIME also reduce the memory requirements. In contrast to pcre2_match(), pcre2_dfa_match() does use recursive - function calls, but only for processing atomic groups, lookaround - assertions, and recursion within the pattern. The original version of - the code used to allocate quite large internal workspace vectors on the + function calls, but only for processing atomic groups, lookaround as- + sertions, and recursion within the pattern. The original version of the + code used to allocate quite large internal workspace vectors on the stack, which caused some problems for some patterns in environments with small stacks. From release 10.32 the code for pcre2_dfa_match() has been re-factored to use heap memory when necessary for internal @@ -9491,8 +9716,8 @@ PROCESSING TIME (a|e|i|o|u). In general, the simplest construction that provides the required behaviour is usually the most efficient. Jeffrey Friedl's book contains a lot of useful general discussion about optimizing regular - expressions for efficient performance. This document contains a few - observations about PCRE2. + expressions for efficient performance. This document contains a few ob- + servations about PCRE2. Using Unicode character properties (the \p, \P, and \X escapes) is slow, because PCRE2 has to use a multi-stage table lookup whenever it @@ -9513,11 +9738,11 @@ PROCESSING TIME option is set, the pattern is implicitly anchored by PCRE2, since it can match only at the start of a subject string. If the pattern has multiple top-level branches, they must all be anchorable. The optimiza- - tion can be disabled by the PCRE2_NO_DOTSTAR_ANCHOR option, and is - automatically disabled if the pattern contains (*PRUNE) or (*SKIP). + tion can be disabled by the PCRE2_NO_DOTSTAR_ANCHOR option, and is au- + tomatically disabled if the pattern contains (*PRUNE) or (*SKIP). - If PCRE2_DOTALL is not set, PCRE2 cannot make this optimization, - because the dot metacharacter does not then match a newline, and if the + If PCRE2_DOTALL is not set, PCRE2 cannot make this optimization, be- + cause the dot metacharacter does not then match a newline, and if the subject string contains newlines, the pattern may match from the char- acter immediately following one of them instead of from the very start. For example, the pattern @@ -9531,8 +9756,8 @@ PROCESSING TIME If you are using such a pattern with subject strings that do not con- tain newlines, the best performance is obtained by setting - PCRE2_DOTALL, or starting the pattern with ^.* or ^.*? to indicate - explicit anchoring. That saves PCRE2 from having to scan along the sub- + PCRE2_DOTALL, or starting the pattern with ^.* or ^.*? to indicate ex- + plicit anchoring. That saves PCRE2 from having to scan along the sub- ject looking for a newline to restart at. Beware of patterns that contain nested indefinite repeats. These can @@ -9546,8 +9771,8 @@ PROCESSING TIME 2, 3, or 4 times, and for each of those cases other than 0 or 4, the + repeats can match different numbers of times.) When the remainder of the pattern is such that the entire match is going to fail, PCRE2 has - in principle to try every possible variation, and this can take an - extremely long time, even for relatively short strings. + in principle to try every possible variation, and this can take an ex- + tremely long time, even for relatively short strings. An optimization catches some of the more simple cases such as @@ -9601,8 +9826,8 @@ PROCESSING TIME matching, and on the amount of heap memory that is used. The default values of the limits are very large, and unlikely ever to operate. They can be changed when PCRE2 is built, and they can also be set when - pcre2_match() or pcre2_dfa_match() is called. For details of these - interfaces, see the pcre2build documentation and the section entitled + pcre2_match() or pcre2_dfa_match() is called. For details of these in- + terfaces, see the pcre2build documentation and the section entitled "The match context" in the pcre2api documentation. The pcre2test test program has a modifier called "find_limits" which, @@ -9652,8 +9877,8 @@ DESCRIPTION This set of functions provides a POSIX-style API for the PCRE2 regular expression 8-bit library. There are no POSIX-style wrappers for PCRE2's - 16-bit and 32-bit libraries. See the pcre2api documentation for a - description of PCRE2's native API, which contains much additional func- + 16-bit and 32-bit libraries. See the pcre2api documentation for a de- + scription of PCRE2's native API, which contains much additional func- tionality. The functions described here are wrapper functions that ultimately call @@ -9714,11 +9939,11 @@ USING THE POSIX FUNCTIONS COMPILING A PATTERN - The function pcre2_regcomp() is called to compile a pattern into an - internal form. By default, the pattern is a C string terminated by a - binary zero (but see REG_PEND below). The preg argument is a pointer to - a regex_t structure that is used as a base for storing information - about the compiled regular expression. (It is also used for input when + The function pcre2_regcomp() is called to compile a pattern into an in- + ternal form. By default, the pattern is a C string terminated by a bi- + nary zero (but see REG_PEND below). The preg argument is a pointer to a + regex_t structure that is used as a base for storing information about + the compiled regular expression. (It is also used for input when REG_PEND is set.) The argument cflags is either zero, or contains one or more of the bits @@ -9754,18 +9979,18 @@ COMPILING A PATTERN REG_NOSUB When a pattern that is compiled with this flag is passed to - pcre2_regexec() for matching, the nmatch and pmatch arguments are - ignored, and no captured strings are returned. Versions of the PCRE - library prior to 10.22 used to set the PCRE2_NO_AUTO_CAPTURE compile - option, but this no longer happens because it disables the use of back- + pcre2_regexec() for matching, the nmatch and pmatch arguments are ig- + nored, and no captured strings are returned. Versions of the PCRE li- + brary prior to 10.22 used to set the PCRE2_NO_AUTO_CAPTURE compile op- + tion, but this no longer happens because it disables the use of back- references. REG_PEND If this option is set, the reg_endp field in the preg structure (which has the type const char *) must be set to point to the character beyond - the end of the pattern before calling pcre2_regcomp(). The pattern - itself may now contain binary zeros, which are treated as data charac- + the end of the pattern before calling pcre2_regcomp(). The pattern it- + self may now contain binary zeros, which are treated as data charac- ters. Without REG_PEND, a binary zero terminates the pattern and the re_endp field is ignored. This is a GNU extension to the POSIX standard and should be used with caution in software intended to be portable to @@ -9792,8 +10017,8 @@ COMPILING A PATTERN Note that REG_UTF is not part of the POSIX standard. In the absence of these flags, no options are passed to the native - function. This means the the regex is compiled with PCRE2 default - semantics. In particular, the way it handles newline characters in the + function. This means the the regex is compiled with PCRE2 default se- + mantics. In particular, the way it handles newline characters in the subject string is the Perl way, not the POSIX way. Note that setting PCRE2_MULTILINE has only some of the effects specified for REG_NEWLINE. It does not affect the way newlines are matched by the dot metacharac- @@ -9802,8 +10027,8 @@ COMPILING A PATTERN The yield of pcre2_regcomp() is zero on success, and non-zero other- wise. The preg structure is filled in on success, and one other member of the structure (as well as re_endp) is public: re_nsub contains the - number of capturing subpatterns in the regular expression. Various - error codes are defined in the header file. + number of capturing subpatterns in the regular expression. Various er- + ror codes are defined in the header file. NOTE: If the yield of pcre2_regcomp() is non-zero, you must not attempt to use the contents of the preg structure. If, for example, you pass it @@ -9844,8 +10069,8 @@ MATCHING NEWLINE CHARACTERS Default POSIX newline handling can be obtained by setting PCRE2_DOTALL and PCRE2_DOLLAR_ENDONLY when calling pcre2_compile() directly, but - there is no way to make PCRE2 behave exactly as for the REG_NEWLINE - action. When using the POSIX API, passing REG_NEWLINE to PCRE2's + there is no way to make PCRE2 behave exactly as for the REG_NEWLINE ac- + tion. When using the POSIX API, passing REG_NEWLINE to PCRE2's pcre2_regcomp() function causes PCRE2_MULTILINE to be passed to pcre2_compile(), and REG_DOTALL passes PCRE2_DOTALL. There is no way to pass PCRE2_DOLLAR_ENDONLY. @@ -9879,15 +10104,15 @@ MATCHING A PATTERN When this option is set, the subject string starts at string + pmatch[0].rm_so and ends at string + pmatch[0].rm_eo, which should - point to the first character beyond the string. There may be binary - zeros within the subject string, and indeed, using REG_STARTEND is the + point to the first character beyond the string. There may be binary ze- + ros within the subject string, and indeed, using REG_STARTEND is the only way to pass a subject string that contains a binary zero. Whatever the value of pmatch[0].rm_so, the offsets of the matched string and any captured substrings are still given relative to the start of string itself. (Before PCRE2 release 10.30 these were given - relative to string + pmatch[0].rm_so, but this differs from other - implementations.) + relative to string + pmatch[0].rm_so, but this differs from other im- + plementations.) This is a BSD extension, compatible with but not specified by IEEE Standard 1003.2 (POSIX.2), and should be used with caution in software @@ -9902,8 +10127,8 @@ MATCHING A PATTERN pcre2_regexec() are ignored (except possibly as input for REG_STAR- TEND). - The value of nmatch may be zero, and the value pmatch may be NULL - (unless REG_STARTEND is set); in both these cases no data about any + The value of nmatch may be zero, and the value pmatch may be NULL (un- + less REG_STARTEND is set); in both these cases no data about any matched strings is returned. Otherwise, the portion of the string that was matched, and also any @@ -9916,9 +10141,9 @@ MATCHING A PATTERN elements relate to the capturing subpatterns of the regular expression. Unused entries in the array have both structure members set to -1. - A successful match yields a zero return; various error codes are - defined in the header file, of which REG_NOMATCH is the "expected" - failure code. + A successful match yields a zero return; various error codes are de- + fined in the header file, of which REG_NOMATCH is the "expected" fail- + ure code. ERROR MESSAGES @@ -9927,8 +10152,8 @@ ERROR MESSAGES pcre2_regcomp() or pcre2_regexec() to a printable message. If preg is not NULL, the error should have arisen from the use of that structure. A message terminated by a binary zero is placed in errbuf. If the buf- - fer is too short, only the first errbuf_size - 1 characters of the - error message are used. The yield of the function is the size of buffer + fer is too short, only the first errbuf_size - 1 characters of the er- + ror message are used. The yield of the function is the size of buffer needed to hold the whole message, including the terminating zero. This value is greater than errbuf_size if the message was truncated. @@ -9937,8 +10162,8 @@ MEMORY USAGE Compiling a regular expression causes memory to be allocated and asso- ciated with the preg structure. The function pcre2_regfree() frees all - such memory, after which preg may no longer be used as a compiled - expression. + such memory, after which preg may no longer be used as a compiled ex- + pression. AUTHOR @@ -9998,8 +10223,8 @@ PCRE2 SAMPLE PROGRAM If PCRE2 is installed elsewhere, you may need to add additional options to the command line. For example, on a Unix-like system that has PCRE2 - installed in /usr/local, you can compile the demonstration program - using a command like this: + installed in /usr/local, you can compile the demonstration program us- + ing a command like this: cc -o pcre2demo -I/usr/local/include pcre2demo.c \ -L/usr/local/lib -lpcre2-8 @@ -10011,8 +10236,8 @@ PCRE2 SAMPLE PROGRAM ./pcre2demo -g 'cat|dog' 'the dog sat on the cat' Note that there is a much more comprehensive test program, called - pcre2test, which supports many more facilities for testing regular - expressions using all three PCRE2 libraries (8-bit, 16-bit, and 32-bit, + pcre2test, which supports many more facilities for testing regular ex- + pressions using all three PCRE2 libraries (8-bit, 16-bit, and 32-bit, though not all three need be installed). The pcre2demo program is pro- vided as a relatively simple coding example. @@ -10148,26 +10373,26 @@ SAVING COMPILED PATTERNS errorcode = fwrite(bytes, 1, bytescount, fd); Note that the serialized data is binary data that may contain any of - the 256 possible byte values. On systems that make a distinction - between binary and non-binary data, be sure that the file is opened for + the 256 possible byte values. On systems that make a distinction be- + tween binary and non-binary data, be sure that the file is opened for binary output. Serializing a set of patterns leaves the original data untouched, so they can still be used for matching. Their memory must eventually be freed in the usual way by calling pcre2_code_free(). When you have fin- ished with the byte stream, it too must be freed by calling pcre2_seri- - alize_free(). If this function is called with a NULL argument, it - returns immediately without doing anything. + alize_free(). If this function is called with a NULL argument, it re- + turns immediately without doing anything. RE-USING PRECOMPILED PATTERNS - In order to re-use a set of saved patterns you must first make the - serialized byte stream available in main memory (for example, by read- - ing from a file). The management of this memory block is up to the - application. You can use the pcre2_serialize_get_number_of_codes() - function to find out how many compiled patterns are in the serialized - data without actually decoding the patterns: + In order to re-use a set of saved patterns you must first make the se- + rialized byte stream available in main memory (for example, by reading + from a file). The management of this memory block is up to the applica- + tion. You can use the pcre2_serialize_get_number_of_codes() function to + find out how many compiled patterns are in the serialized data without + actually decoding the patterns: uint8_t *bytes = <serialized data>; int32_t number_of_codes = pcre2_serialize_get_number_of_codes(bytes); @@ -10175,8 +10400,8 @@ RE-USING PRECOMPILED PATTERNS The pcre2_serialize_decode() function reads a byte stream and recreates the compiled patterns in new memory blocks, setting pointers to them in a vector. The first two arguments are a pointer to a suitable vector - and its length, and the third argument points to a byte stream. The - final argument is a pointer to a general context, which can be used to + and its length, and the third argument points to a byte stream. The fi- + nal argument is a pointer to a general context, which can be used to specify custom memory mangagement functions for the decoded patterns. If this argument is NULL, malloc() and free() are used. After deserial- ization, the byte stream is no longer needed and can be discarded. @@ -10188,9 +10413,9 @@ RE-USING PRECOMPILED PATTERNS pcre2_serialize_decode(list_of_codes, 2, bytes, NULL); If the vector is not large enough for all the patterns in the byte - stream, it is filled with those that fit, and the remainder are - ignored. The yield of the function is the number of decoded patterns, - or one of the following negative error codes: + stream, it is filled with those that fit, and the remainder are ig- + nored. The yield of the function is the number of decoded patterns, or + one of the following negative error codes: PCRE2_ERROR_BADDATA second argument is zero or less PCRE2_ERROR_BADMAGIC mismatch of id bytes in the data @@ -10204,9 +10429,9 @@ RE-USING PRECOMPILED PATTERNS Decoded patterns can be used for matching in the usual way, and must be freed by calling pcre2_code_free(). However, be aware that there is a - potential race issue if you are using multiple patterns that were - decoded from a single byte stream in a multithreaded application. A - single copy of the character tables is used by all the decoded patterns + potential race issue if you are using multiple patterns that were de- + coded from a single byte stream in a multithreaded application. A sin- + gle copy of the character tables is used by all the decoded patterns and a reference count is used to arrange for its memory to be automati- cally freed when the last pattern is freed, but there is no locking on this reference count. Therefore, if you want to call pcre2_code_free() @@ -10289,8 +10514,8 @@ ESCAPED CHARACTERS Note that \0dd is always an octal code. The treatment of backslash fol- lowed by a non-zero digit is complicated; for details see the section - "Non-printing characters" in the pcre2pattern documentation, where - details of escape processing in EBCDIC environments are also given. + "Non-printing characters" in the pcre2pattern documentation, where de- + tails of escape processing in EBCDIC environments are also given. \N{U+hh..} is synonymous with \x{hh..} in PCRE2 but is not supported in EBCDIC environments. Note that \N not followed by an opening curly bracket has a different meaning (see below). @@ -10396,26 +10621,28 @@ SCRIPT NAMES FOR \p AND \P Adlam, Ahom, Anatolian_Hieroglyphs, Arabic, Armenian, Avestan, Bali- nese, Bamum, Bassa_Vah, Batak, Bengali, Bhaiksuki, Bopomofo, Brahmi, Braille, Buginese, Buhid, Canadian_Aboriginal, Carian, Caucasian_Alba- - nian, Chakma, Cham, Cherokee, Common, Coptic, Cuneiform, Cypriot, - Cyrillic, Deseret, Devanagari, Dogra, Duployan, Egyptian_Hieroglyphs, - Elbasan, Ethiopic, Georgian, Glagolitic, Gothic, Grantha, Greek, - Gujarati, Gunjala_Gondi, Gurmukhi, Han, Hangul, Hanifi_Rohingya, - Hanunoo, Hatran, Hebrew, Hiragana, Imperial_Aramaic, Inherited, - Inscriptional_Pahlavi, Inscriptional_Parthian, Javanese, Kaithi, Kan- - nada, Katakana, Kayah_Li, Kharoshthi, Khmer, Khojki, Khudawadi, Lao, - Latin, Lepcha, Limbu, Linear_A, Linear_B, Lisu, Lycian, Lydian, Maha- - jani, Makasar, Malayalam, Mandaic, Manichaean, Marchen, Masaram_Gondi, - Medefaidrin, Meetei_Mayek, Mende_Kikakui, Meroitic_Cursive, - Meroitic_Hieroglyphs, Miao, Modi, Mongolian, Mro, Multani, Myanmar, - Nabataean, New_Tai_Lue, Newa, Nko, Nushu, Ogham, Ol_Chiki, Old_Hungar- - ian, Old_Italic, Old_North_Arabian, Old_Permic, Old_Persian, Old_Sog- - dian, Old_South_Arabian, Old_Turkic, Oriya, Osage, Osmanya, - Pahawh_Hmong, Palmyrene, Pau_Cin_Hau, Phags_Pa, Phoenician, - Psalter_Pahlavi, Rejang, Runic, Samaritan, Saurashtra, Sharada, Sha- - vian, Siddham, SignWriting, Sinhala, Sogdian, Sora_Sompeng, Soyombo, - Sundanese, Syloti_Nagri, Syriac, Tagalog, Tagbanwa, Tai_Le, Tai_Tham, - Tai_Viet, Takri, Tamil, Tangut, Telugu, Thaana, Thai, Tibetan, Tifi- - nagh, Tirhuta, Ugaritic, Vai, Warang_Citi, Yi, Zanabazar_Square. + nian, Chakma, Cham, Cherokee, Chorasmian, Common, Coptic, Cuneiform, + Cypriot, Cyrillic, Deseret, Devanagari, Dives_Akuru, Dogra, Duployan, + Egyptian_Hieroglyphs, Elbasan, Elymaic, Ethiopic, Georgian, Glagolitic, + Gothic, Grantha, Greek, Gujarati, Gunjala_Gondi, Gurmukhi, Han, Hangul, + Hanifi_Rohingya, Hanunoo, Hatran, Hebrew, Hiragana, Imperial_Aramaic, + Inherited, Inscriptional_Pahlavi, Inscriptional_Parthian, Javanese, + Kaithi, Kannada, Katakana, Kayah_Li, Kharoshthi, Khitan_Small_Script, + Khmer, Khojki, Khudawadi, Lao, Latin, Lepcha, Limbu, Linear_A, Lin- + ear_B, Lisu, Lycian, Lydian, Mahajani, Makasar, Malayalam, Mandaic, + Manichaean, Marchen, Masaram_Gondi, Medefaidrin, Meetei_Mayek, + Mende_Kikakui, Meroitic_Cursive, Meroitic_Hieroglyphs, Miao, Modi, Mon- + golian, Mro, Multani, Myanmar, Nabataean, Nandinagari, New_Tai_Lue, + Newa, Nko, Nushu, Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungar- + ian, Old_Italic, Old_North_Arabian, Old_Permic, Old_Persian, Old_Sog- + dian, Old_South_Arabian, Old_Turkic, Oriya, Osage, Osmanya, Pa- + hawh_Hmong, Palmyrene, Pau_Cin_Hau, Phags_Pa, Phoenician, + Psalter_Pahlavi, Rejang, Runic, Samaritan, Saurashtra, Sharada, Sha- + vian, Siddham, SignWriting, Sinhala, Sogdian, Sora_Sompeng, Soyombo, + Sundanese, Syloti_Nagri, Syriac, Tagalog, Tagbanwa, Tai_Le, Tai_Tham, + Tai_Viet, Takri, Tamil, Tangut, Telugu, Thaana, Thai, Tibetan, Tifi- + nagh, Tirhuta, Ugaritic, Vai, Wancho, Warang_Citi, Yezidi, Yi, Zan- + abazar_Square. CHARACTER CLASSES @@ -10526,7 +10753,7 @@ OPTION SETTING the end of the group. (?i) caseless - (?J) allow duplicate names + (?J) allow duplicate named groups (?m) multiline (?n) no auto capture (?s) single line (dotall) @@ -10542,9 +10769,9 @@ OPTION SETTING for example (?^in). An option setting may appear at the start of a non- capture group, for example (?i:...). - The following are recognized only at the very start of a pattern or - after one of the newline or \R options with similar syntax. More than - one of them may appear. For the first three, d is a decimal number. + The following are recognized only at the very start of a pattern or af- + ter one of the newline or \R options with similar syntax. More than one + of them may appear. For the first three, d is a decimal number. (*LIMIT_DEPTH=d) set the backtracking limit to d (*LIMIT_HEAP=d) set the heap size limit to d * 1024 bytes @@ -10568,8 +10795,8 @@ OPTION SETTING NEWLINE CONVENTION - These are recognized only at the very start of the pattern or after - option settings with a similar syntax. + These are recognized only at the very start of the pattern or after op- + tion settings with a similar syntax. (*CR) carriage return only (*LF) linefeed only @@ -10581,8 +10808,8 @@ NEWLINE CONVENTION WHAT \R MATCHES - These are recognized only at the very start of the pattern or after - option setting with a similar syntax. + These are recognized only at the very start of the pattern or after op- + tion setting with a similar syntax. (*BSR_ANYCRLF) CR, LF, or CRLF (*BSR_UNICODE) any Unicode newline sequence @@ -10609,6 +10836,19 @@ LOOKAHEAD AND LOOKBEHIND ASSERTIONS Each top-level branch of a lookbehind must be of a fixed length. +NON-ATOMIC LOOKAROUND ASSERTIONS + + These assertions are specific to PCRE2 and are not Perl-compatible. + + (?*...) ) + (*napla:...) ) synonyms + (*non_atomic_positive_lookahead:...) ) + + (?<*...) ) + (*naplb:...) ) synonyms + (*non_atomic_positive_lookbehind:...) ) + + SCRIPT RUNS (*script_run:...) ) script run, can be backtracked into @@ -10730,7 +10970,7 @@ AUTHOR REVISION - Last updated: 11 February 2019 + Last updated: 28 December 2019 Copyright (c) 1997-2019 University of Cambridge. ------------------------------------------------------------------------------ @@ -10744,21 +10984,29 @@ NAME UNICODE AND UTF SUPPORT - When PCRE2 is built with Unicode support (which is the default), it has - knowledge of Unicode character properties and can process text strings - in UTF-8, UTF-16, or UTF-32 format (depending on the code unit width). - However, by default, PCRE2 assumes that one code unit is one character. - To process a pattern as a UTF string, where a character may require - more than one code unit, you must call pcre2_compile() with the - PCRE2_UTF option flag, or the pattern must start with the sequence - (*UTF). When either of these is the case, both the pattern and any sub- - ject strings that are matched against it are treated as UTF strings - instead of strings of individual one-code-unit characters. There are - also some other changes to the way characters are handled, as docu- - mented below. + PCRE2 is normally built with Unicode support, though if you do not need + it, you can build it without, in which case the library will be + smaller. With Unicode support, PCRE2 has knowledge of Unicode character + properties and can process strings of text in UTF-8, UTF-16, and UTF-32 + format (depending on the code unit width), but this is not the default. + Unless specifically requested, PCRE2 treats each code unit in a string + as one character. + + There are two ways of telling PCRE2 to switch to UTF mode, where char- + acters may consist of more than one code unit and the range of values + is constrained. The program can call pcre2_compile() with the PCRE2_UTF + option, or the pattern may start with the sequence (*UTF). However, + the latter facility can be locked out by the PCRE2_NEVER_UTF option. + That is, the programmer can prevent the supplier of the pattern from + switching to UTF mode. - If you do not need Unicode support you can build PCRE2 without it, in - which case the library will be smaller. + Note that the PCRE2_MATCH_INVALID_UTF option (see below) forces + PCRE2_UTF to be set. + + In UTF mode, both the pattern and any subject strings that are matched + against it are treated as UTF strings instead of strings of individual + one-code-unit characters. There are also some other changes to the way + characters are handled, as documented below. UNICODE PROPERTY SUPPORT @@ -10785,18 +11033,18 @@ WIDE CHARACTERS AND UTF MODES The escape sequence \N{U+<hex digits>} is recognized as another way of specifying a Unicode character by code point in a UTF mode. It is not - allowed in non-UTF modes. + allowed in non-UTF mode. - In UTF modes, repeat quantifiers apply to complete UTF characters, not + In UTF mode, repeat quantifiers apply to complete UTF characters, not to individual code units. - In UTF modes, the dot metacharacter matches one UTF character instead - of a single code unit. + In UTF mode, the dot metacharacter matches one UTF character instead of + a single code unit. - In UTF modes, capture group names are not restricted to ASCII, and may + In UTF mode, capture group names are not restricted to ASCII, and may contain any Unicode letters and decimal digits, as well as underscore. - The escape sequence \C can be used to match a single code unit in a UTF + The escape sequence \C can be used to match a single code unit in UTF mode, but its use can lead to some strange effects because it breaks up multi-unit characters (see the description of \C in the pcre2pattern documentation). For this reason, there is a build-time option that dis- @@ -10810,39 +11058,41 @@ WIDE CHARACTERS AND UTF MODES modes provokes a match-time error. Also, the JIT optimization does not support \C in these modes. If JIT optimization is requested for a UTF-8 or UTF-16 pattern that contains \C, it will not succeed, and so when - pcre2_match() is called, the matching will be carried out by the normal - interpretive function. + pcre2_match() is called, the matching will be carried out by the inter- + pretive function. The character escapes \b, \B, \d, \D, \s, \S, \w, and \W correctly test characters of any code value, but, by default, the characters that PCRE2 recognizes as digits, spaces, or word characters remain the same - set as in non-UTF mode, all with code points less than 256. This - remains true even when PCRE2 is built to include Unicode support, - because to do otherwise would slow down matching in many common cases. - Note that this also applies to \b and \B, because they are defined in - terms of \w and \W. If you want to test for a wider sense of, say, - "digit", you can use explicit Unicode property tests such as \p{Nd}. - Alternatively, if you set the PCRE2_UCP option, the way that the char- - acter escapes work is changed so that Unicode properties are used to - determine which characters match. There are more details in the section - on generic character types in the pcre2pattern documentation. + set as in non-UTF mode, all with code points less than 256. This re- + mains true even when PCRE2 is built to include Unicode support, because + to do otherwise would slow down matching in many common cases. Note + that this also applies to \b and \B, because they are defined in terms + of \w and \W. If you want to test for a wider sense of, say, "digit", + you can use explicit Unicode property tests such as \p{Nd}. Alterna- + tively, if you set the PCRE2_UCP option, the way that the character es- + capes work is changed so that Unicode properties are used to determine + which characters match. There are more details in the section on + generic character types in the pcre2pattern documentation. Similarly, characters that match the POSIX named character classes are all low-valued characters, unless the PCRE2_UCP option is set. - However, the special horizontal and vertical white space matching - escapes (\h, \H, \v, and \V) do match all the appropriate Unicode char- - acters, whether or not PCRE2_UCP is set. + However, the special horizontal and vertical white space matching es- + capes (\h, \H, \v, and \V) do match all the appropriate Unicode charac- + ters, whether or not PCRE2_UCP is set. -CASE-EQUIVALENCE IN UTF MODES +UNICODE CASE-EQUIVALENCE - Case-insensitive matching in a UTF mode makes use of Unicode properties - except for characters whose code points are less than 128 and that have - at most two case-equivalent values. For these, a direct table lookup is - used for speed. A few Unicode characters such as Greek sigma have more - than two code points that are case-equivalent, and these are treated as - such. + If either PCRE2_UTF or PCRE2_UCP is set, upper/lower case processing + makes use of Unicode properties except for characters whose code points + are less than 128 and that have at most two case-equivalent values. For + these, a direct table lookup is used for speed. A few Unicode charac- + ters such as Greek sigma have more than two code points that are case- + equivalent, and these are treated specially. Setting PCRE2_UCP without + PCRE2_UTF allows Unicode-style case processing for non-UTF character + encodings such as UCS-2. SCRIPT RUNS @@ -10877,12 +11127,12 @@ SCRIPT RUNS are only normally used with a small number of scripts. For example, U+102E0 (Coptic Epact thousands mark) is used only with Arabic and Cop- tic. In order to make it possible to check this, a Unicode property - called Script Extension exists. Its value is a list of scripts that - apply to the character. For the majority of characters, the list con- - tains just one script, the same one as the Script property. However, - for characters such as U+102E0 more than one Script is listed. There - are also some Common characters that have a single, non-Common script - in their Script Extension list. + called Script Extension exists. Its value is a list of scripts that ap- + ply to the character. For the majority of characters, the list contains + just one script, the same one as the Script property. However, for + characters such as U+102E0 more than one Script is listed. There are + also some Common characters that have a single, non-Common script in + their Script Extension list. The next section describes the basic rules for deciding whether a given string of characters is a script run. Note, however, that there are @@ -10919,8 +11169,8 @@ SCRIPT RUNS The first has the Script Extension list Arabic, Hanifi Rohingya, Syr- iac, and Thaana; the second has just Arabic and Hanifi Rohingya. Both - of them could appear in script runs of either Arabic or Hanifi - Rohingya. The first could also appear in Syriac or Thaana script runs, + of them could appear in script runs of either Arabic or Hanifi Ro- + hingya. The first could also appear in Syriac or Thaana script runs, but the second could not. The Chinese Han script @@ -10934,16 +11184,16 @@ SCRIPT RUNS gana, Katakana, and Han, or a mixture of Hangul and Han, or a mixture of Bopomofo and Han, but not, for example, a mixture of Hangul and Bopomofo and Han. PCRE2 (like Perl) follows Unicode's Technical Stan- - dard 39 ("Unicode Security Mechanisms", http://uni- - code.org/reports/tr39/) in allowing such mixtures. + dard 39 ("Unicode Security Mechanisms", http://unicode.org/re- + ports/tr39/) in allowing such mixtures. Decimal digits Unicode contains many sets of 10 decimal digits in different scripts, and some scripts (including the Common script) contain more than one set. Some of these decimal digits them are visually indistinguishable - from the common ASCII digits. In addition to the script checking - described above, if a script run contains any decimal digits, they must + from the common ASCII digits. In addition to the script checking de- + scribed above, if a script run contains any decimal digits, they must all come from the same set of 10 adjacent characters. @@ -10951,9 +11201,9 @@ VALIDITY OF UTF STRINGS When the PCRE2_UTF option is set, the strings passed as patterns and subjects are (by default) checked for validity on entry to the relevant - functions. If an invalid UTF string is passed, an negative error code - is returned. The code unit offset to the offending character can be - extracted from the match data block by calling pcre2_get_startchar(), + functions. If an invalid UTF string is passed, a negative error code is + returned. The code unit offset to the offending character can be ex- + tracted from the match data block by calling pcre2_get_startchar(), which is used for this purpose after a UTF error. In some situations, you may already know that your strings are valid, @@ -10964,53 +11214,54 @@ VALIDITY OF UTF STRINGS it is given (respectively) contains only valid UTF code unit sequences. If you pass an invalid UTF string when PCRE2_NO_UTF_CHECK is set, the - result is usually undefined and your program may crash or loop indefi- - nitely. There is, however, one mode of matching that can handle invalid - UTF subject strings. This is matching via the JIT optimization using - the PCRE2_JIT_INVALID_UTF option when calling pcre2_jit_compile(). For - details, see the pcre2jit documentation. - - Passing PCRE2_NO_UTF_CHECK to pcre2_compile() just disables the check - for the pattern; it does not also apply to subject strings. If you want - to disable the check for a subject string you must pass this same - option to pcre2_match() or pcre2_dfa_match(). + result is undefined and your program may crash or loop indefinitely or + give incorrect results. There is, however, one mode of matching that + can handle invalid UTF subject strings. This is enabled by passing + PCRE2_MATCH_INVALID_UTF to pcre2_compile() and is discussed below in + the next section. The rest of this section covers the case when + PCRE2_MATCH_INVALID_UTF is not set. + + Passing PCRE2_NO_UTF_CHECK to pcre2_compile() just disables the UTF + check for the pattern; it does not also apply to subject strings. If + you want to disable the check for a subject string you must pass this + same option to pcre2_match() or pcre2_dfa_match(). UTF-16 and UTF-32 strings can indicate their endianness by special code - knows as a byte-order mark (BOM). The PCRE2 functions do not handle + knows as a byte-order mark (BOM). The PCRE2 functions do not handle this, expecting strings to be in host byte order. - Unless PCRE2_NO_UTF_CHECK is set, a UTF string is checked before any + Unless PCRE2_NO_UTF_CHECK is set, a UTF string is checked before any other processing takes place. In the case of pcre2_match() and - pcre2_dfa_match() calls with a non-zero starting offset, the check is + pcre2_dfa_match() calls with a non-zero starting offset, the check is applied only to that part of the subject that could be inspected during - matching, and there is a check that the starting offset points to the - first code unit of a character or to the end of the subject. If there - are no lookbehind assertions in the pattern, the check starts at the - starting offset. Otherwise, it starts at the length of the longest - lookbehind before the starting offset, or at the start of the subject - if there are not that many characters before the starting offset. Note + matching, and there is a check that the starting offset points to the + first code unit of a character or to the end of the subject. If there + are no lookbehind assertions in the pattern, the check starts at the + starting offset. Otherwise, it starts at the length of the longest + lookbehind before the starting offset, or at the start of the subject + if there are not that many characters before the starting offset. Note that the sequences \b and \B are one-character lookbehinds. - In addition to checking the format of the string, there is a check to + In addition to checking the format of the string, there is a check to ensure that all code points lie in the range U+0 to U+10FFFF, excluding - the surrogate area. The so-called "non-character" code points are not + the surrogate area. The so-called "non-character" code points are not excluded because Unicode corrigendum #9 makes it clear that they should not be. - Characters in the "Surrogate Area" of Unicode are reserved for use by - UTF-16, where they are used in pairs to encode code points with values - greater than 0xFFFF. The code points that are encoded by UTF-16 pairs - are available independently in the UTF-8 and UTF-32 encodings. (In - other words, the whole surrogate thing is a fudge for UTF-16 which - unfortunately messes up UTF-8 and UTF-32.) - - Setting PCRE2_NO_UTF_CHECK at compile time does not disable the error - that is given if an escape sequence for an invalid Unicode code point - is encountered in the pattern. If you want to allow escape sequences - such as \x{d800} (a surrogate code point) you can set the - PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES extra option. However, this is pos- - sible only in UTF-8 and UTF-32 modes, because these values are not rep- - resentable in UTF-16. + Characters in the "Surrogate Area" of Unicode are reserved for use by + UTF-16, where they are used in pairs to encode code points with values + greater than 0xFFFF. The code points that are encoded by UTF-16 pairs + are available independently in the UTF-8 and UTF-32 encodings. (In + other words, the whole surrogate thing is a fudge for UTF-16 which un- + fortunately messes up UTF-8 and UTF-32.) + + Setting PCRE2_NO_UTF_CHECK at compile time does not disable the error + that is given if an escape sequence for an invalid Unicode code point + is encountered in the pattern. If you want to allow escape sequences + such as \x{d800} (a surrogate code point) you can set the PCRE2_EX- + TRA_ALLOW_SURROGATE_ESCAPES extra option. However, this is possible + only in UTF-8 and UTF-32 modes, because these values are not repre- + sentable in UTF-16. Errors in UTF-8 strings @@ -11022,10 +11273,10 @@ VALIDITY OF UTF STRINGS PCRE2_ERROR_UTF8_ERR4 PCRE2_ERROR_UTF8_ERR5 - The string ends with a truncated UTF-8 character; the code specifies - how many bytes are missing (1 to 5). Although RFC 3629 restricts UTF-8 - characters to be no longer than 4 bytes, the encoding scheme (origi- - nally defined by RFC 2279) allows for up to 6 bytes, and this is + The string ends with a truncated UTF-8 character; the code specifies + how many bytes are missing (1 to 5). Although RFC 3629 restricts UTF-8 + characters to be no longer than 4 bytes, the encoding scheme (origi- + nally defined by RFC 2279) allows for up to 6 bytes, and this is checked first; hence the possibility of 4 or 5 missing bytes. PCRE2_ERROR_UTF8_ERR6 @@ -11035,24 +11286,24 @@ VALIDITY OF UTF STRINGS PCRE2_ERROR_UTF8_ERR10 The two most significant bits of the 2nd, 3rd, 4th, 5th, or 6th byte of - the character do not have the binary value 0b10 (that is, either the + the character do not have the binary value 0b10 (that is, either the most significant bit is 0, or the next bit is 1). PCRE2_ERROR_UTF8_ERR11 PCRE2_ERROR_UTF8_ERR12 - A character that is valid by the RFC 2279 rules is either 5 or 6 bytes + A character that is valid by the RFC 2279 rules is either 5 or 6 bytes long; these code points are excluded by RFC 3629. PCRE2_ERROR_UTF8_ERR13 - A 4-byte character has a value greater than 0x10fff; these code points + A 4-byte character has a value greater than 0x10ffff; these code points are excluded by RFC 3629. PCRE2_ERROR_UTF8_ERR14 - A 3-byte character has a value in the range 0xd800 to 0xdfff; this - range of code points are reserved by RFC 3629 for use with UTF-16, and + A 3-byte character has a value in the range 0xd800 to 0xdfff; this + range of code points are reserved by RFC 3629 for use with UTF-16, and so are excluded from UTF-8. PCRE2_ERROR_UTF8_ERR15 @@ -11061,26 +11312,26 @@ VALIDITY OF UTF STRINGS PCRE2_ERROR_UTF8_ERR18 PCRE2_ERROR_UTF8_ERR19 - A 2-, 3-, 4-, 5-, or 6-byte character is "overlong", that is, it codes - for a value that can be represented by fewer bytes, which is invalid. - For example, the two bytes 0xc0, 0xae give the value 0x2e, whose cor- + A 2-, 3-, 4-, 5-, or 6-byte character is "overlong", that is, it codes + for a value that can be represented by fewer bytes, which is invalid. + For example, the two bytes 0xc0, 0xae give the value 0x2e, whose cor- rect coding uses just one byte. PCRE2_ERROR_UTF8_ERR20 The two most significant bits of the first byte of a character have the - binary value 0b10 (that is, the most significant bit is 1 and the sec- - ond is 0). Such a byte can only validly occur as the second or subse- + binary value 0b10 (that is, the most significant bit is 1 and the sec- + ond is 0). Such a byte can only validly occur as the second or subse- quent byte of a multi-byte character. PCRE2_ERROR_UTF8_ERR21 - The first byte of a character has the value 0xfe or 0xff. These values + The first byte of a character has the value 0xfe or 0xff. These values can never occur in a valid UTF-8 string. Errors in UTF-16 strings - The following negative error codes are given for invalid UTF-16 + The following negative error codes are given for invalid UTF-16 strings: PCRE2_ERROR_UTF16_ERR1 Missing low surrogate at end of string @@ -11090,13 +11341,61 @@ VALIDITY OF UTF STRINGS Errors in UTF-32 strings - The following negative error codes are given for invalid UTF-32 + The following negative error codes are given for invalid UTF-32 strings: PCRE2_ERROR_UTF32_ERR1 Surrogate character (0xd800 to 0xdfff) PCRE2_ERROR_UTF32_ERR2 Code point is greater than 0x10ffff +MATCHING IN INVALID UTF STRINGS + + You can run pattern matches on subject strings that may contain invalid + UTF sequences if you call pcre2_compile() with the PCRE2_MATCH_IN- + VALID_UTF option. This is supported by pcre2_match(), including JIT + matching, but not by pcre2_dfa_match(). When PCRE2_MATCH_INVALID_UTF is + set, it forces PCRE2_UTF to be set as well. Note, however, that the + pattern itself must be a valid UTF string. + + Setting PCRE2_MATCH_INVALID_UTF does not affect what pcre2_compile() + generates, but if pcre2_jit_compile() is subsequently called, it does + generate different code. If JIT is not used, the option affects the be- + haviour of the interpretive code in pcre2_match(). When PCRE2_MATCH_IN- + VALID_UTF is set at compile time, PCRE2_NO_UTF_CHECK is ignored at + match time. + + In this mode, an invalid code unit sequence in the subject never + matches any pattern item. It does not match dot, it does not match + \p{Any}, it does not even match negative items such as [^X]. A lookbe- + hind assertion fails if it encounters an invalid sequence while moving + the current point backwards. In other words, an invalid UTF code unit + sequence acts as a barrier which no match can cross. + + You can also think of this as the subject being split up into fragments + of valid UTF, delimited internally by invalid code unit sequences. The + pattern is matched fragment by fragment. The result of a successful + match, however, is given as code unit offsets in the entire subject + string in the usual way. There are a few points to consider: + + The internal boundaries are not interpreted as the beginnings or ends + of lines and so do not match circumflex or dollar characters in the + pattern. + + If pcre2_match() is called with an offset that points to an invalid + UTF-sequence, that sequence is skipped, and the match starts at the + next valid UTF character, or the end of the subject. + + At internal fragment boundaries, \b and \B behave in the same way as at + the beginning and end of the subject. For example, a sequence such as + \bWORD\b would match an instance of WORD that is surrounded by invalid + UTF code units. + + Using PCRE2_MATCH_INVALID_UTF, an application can run matches on arbi- + trary data, knowing that any matched strings that are returned are + valid UTF. This can be useful when searching for UTF text in executable + or other binary files. + + AUTHOR Philip Hazel @@ -11106,8 +11405,8 @@ AUTHOR REVISION - Last updated: 06 March 2019 - Copyright (c) 1997-2019 University of Cambridge. + Last updated: 23 February 2020 + Copyright (c) 1997-2020 University of Cambridge. ------------------------------------------------------------------------------ diff --git a/dist2/doc/pcre2_compile.3 b/dist2/doc/pcre2_compile.3 index b23bf46d..58a60c1d 100644 --- a/dist2/doc/pcre2_compile.3 +++ b/dist2/doc/pcre2_compile.3 @@ -1,4 +1,4 @@ -.TH PCRE2_COMPILE 3 "11 February 2019" "PCRE2 10.33" +.TH PCRE2_COMPILE 3 "23 May 2019" "PCRE2 10.34" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH SYNOPSIS @@ -53,6 +53,7 @@ The option bits are: PCRE2_EXTENDED Ignore white space and # comments PCRE2_FIRSTLINE Force matching to be before newline PCRE2_LITERAL Pattern characters are all literal + PCRE2_MATCH_INVALID_UTF Enable support for matching invalid UTF PCRE2_MATCH_UNSET_BACKREF Match unset backreferences PCRE2_MULTILINE ^ and $ match newlines within data PCRE2_NEVER_BACKSLASH_C Lock out the use of \eC in patterns diff --git a/dist2/doc/pcre2_get_match_data_size.3 b/dist2/doc/pcre2_get_match_data_size.3 new file mode 100644 index 00000000..cf5fa5e6 --- /dev/null +++ b/dist2/doc/pcre2_get_match_data_size.3 @@ -0,0 +1,27 @@ +.TH PCRE2_GET_MATCH_DATA_SIZE 3 "16 July 2019" "PCRE2 10.34" +.SH NAME +PCRE2 - Perl-compatible regular expressions (revised API) +.SH SYNOPSIS +.rs +.sp +.B #include <pcre2.h> +.PP +.nf +.B PCRE2_SIZE pcre2_get_match_data_size(pcre2_match_data *\fImatch_data\fP); +.fi +. +.SH DESCRIPTION +.rs +.sp +This function returns the size, in bytes, of the match data block that is its +argument. +.P +There is a complete description of the PCRE2 native API in the +.\" HREF +\fBpcre2api\fP +.\" +page and a description of the POSIX API in the +.\" HREF +\fBpcre2posix\fP +.\" +page. diff --git a/dist2/doc/pcre2_jit_compile.3 b/dist2/doc/pcre2_jit_compile.3 index 23dd2d2d..6cc17880 100644 --- a/dist2/doc/pcre2_jit_compile.3 +++ b/dist2/doc/pcre2_jit_compile.3 @@ -1,4 +1,4 @@ -.TH PCRE2_JIT_COMPILE 3 "06 March 2019" "PCRE2 10.33" +.TH PCRE2_JIT_COMPILE 3 "29 July 2019" "PCRE2 10.34" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH SYNOPSIS @@ -29,11 +29,16 @@ bits: PCRE2_JIT_COMPLETE compile code for full matching PCRE2_JIT_PARTIAL_SOFT compile code for soft partial matching PCRE2_JIT_PARTIAL_HARD compile code for hard partial matching - PCRE2_JIT_INVALID_UTF compile code to handle invalid UTF .sp +There is also an obsolete option called PCRE2_JIT_INVALID_UTF, which has been +superseded by the \fBpcre2_compile()\fP option PCRE2_MATCH_INVALID_UTF. The old +option is deprecated and may be removed in the future. +.P The yield of the function is 0 for success, or a negative error code otherwise. In particular, PCRE2_ERROR_JIT_BADOPTION is returned if JIT is not supported or -if an unknown bit is set in \fIoptions\fP. +if an unknown bit is set in \fIoptions\fP. The function can also return +PCRE2_ERROR_NOMEMORY if JIT is unable to allocate executable memory for the +compiler, even if it was because of a system security restriction. .P There is a complete description of the PCRE2 native API in the .\" HREF diff --git a/dist2/doc/pcre2_jit_free_unused_memory.3 b/dist2/doc/pcre2_jit_free_unused_memory.3 index bf050c8a..183bba0a 100644 --- a/dist2/doc/pcre2_jit_free_unused_memory.3 +++ b/dist2/doc/pcre2_jit_free_unused_memory.3 @@ -17,7 +17,7 @@ This function frees unused JIT executable memory. The argument is a general context, for custom memory management, or NULL for standard memory management. JIT memory allocation retains some memory in order to improve future JIT compilation speed. In low memory conditions, -\fBpcre2_jit_free_unused_memory()\fB can be used to cause this memory to be +\fBpcre2_jit_free_unused_memory()\fP can be used to cause this memory to be freed. .P There is a complete description of the PCRE2 native API in the diff --git a/dist2/doc/pcre2_jit_match.3 b/dist2/doc/pcre2_jit_match.3 index b0cc197d..5877fcba 100644 --- a/dist2/doc/pcre2_jit_match.3 +++ b/dist2/doc/pcre2_jit_match.3 @@ -1,4 +1,4 @@ -.TH PCRE2_JIT_MATCH 3 "03 November 2014" "PCRE2 10.0" +.TH PCRE2_JIT_MATCH 3 "11 February 2020" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH SYNOPSIS @@ -22,8 +22,10 @@ algorithm that is similar to Perl's. It is a "fast path" interface to JIT, and it bypasses some of the sanity checks that \fBpcre2_match()\fP applies. Its arguments are exactly the same as for .\" HREF -\fBpcre2_match()\fP. +\fBpcre2_match()\fP, .\" +except that the subject string must be specified with a length; +PCRE2_ZERO_TERMINATED is not supported. .P The supported options are PCRE2_NOTBOL, PCRE2_NOTEOL, PCRE2_NOTEMPTY, PCRE2_NOTEMPTY_ATSTART, PCRE2_PARTIAL_HARD, and PCRE2_PARTIAL_SOFT. Unsupported diff --git a/dist2/doc/pcre2_maketables.3 b/dist2/doc/pcre2_maketables.3 index 740954b9..7dc8438b 100644 --- a/dist2/doc/pcre2_maketables.3 +++ b/dist2/doc/pcre2_maketables.3 @@ -7,7 +7,7 @@ PCRE2 - Perl-compatible regular expressions (revised API) .B #include <pcre2.h> .PP .SM -.B const unsigned char *pcre2_maketables(pcre2_general_context *\fIgcontext\fP); +.B const uint8_t *pcre2_maketables(pcre2_general_context *\fIgcontext\fP); . .SH DESCRIPTION .rs diff --git a/dist2/doc/pcre2_maketables_free.3 b/dist2/doc/pcre2_maketables_free.3 new file mode 100644 index 00000000..07986b97 --- /dev/null +++ b/dist2/doc/pcre2_maketables_free.3 @@ -0,0 +1,31 @@ +.TH PCRE2_MAKETABLES_FREE 3 "02 September 2019" "PCRE2 10.34" +.SH NAME +PCRE2 - Perl-compatible regular expressions (revised API) +.SH SYNOPSIS +.rs +.sp +.B #include <pcre2.h> +.PP +.nf +.B void pcre2_maketables_free(pcre2_general_context *\fIgcontext\fP, +.B " const uint8_t *\fItables\fP);" +.fi +. +.SH DESCRIPTION +.rs +.sp +This function discards a set of character tables that were created by a call +to +.\" HREF +\fBpcre2_maketables()\fP. +.\" +.P +The \fIgcontext\fP parameter should match what was used in that call to +account for any custom allocators that might be in use; if it is NULL +the system \fBfree()\fP is used. +.P +There is a complete description of the PCRE2 native API in the +.\" HREF +\fBpcre2api\fP +.\" +page. diff --git a/dist2/doc/pcre2_set_character_tables.3 b/dist2/doc/pcre2_set_character_tables.3 index 1ede4f91..1ca41347 100644 --- a/dist2/doc/pcre2_set_character_tables.3 +++ b/dist2/doc/pcre2_set_character_tables.3 @@ -1,4 +1,4 @@ -.TH PCRE2_SET_CHARACTER_TABLES 3 "22 October 2014" "PCRE2 10.00" +.TH PCRE2_SET_CHARACTER_TABLES 3 "20 March 2020" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH SYNOPSIS @@ -8,16 +8,21 @@ PCRE2 - Perl-compatible regular expressions (revised API) .PP .nf .B int pcre2_set_character_tables(pcre2_compile_context *\fIccontext\fP, -.B " const unsigned char *\fItables\fP);" +.B " const uint8_t *\fItables\fP);" .fi . .SH DESCRIPTION .rs .sp This function sets a pointer to custom character tables within a compile -context. The second argument must be the result of a call to -\fBpcre2_maketables()\fP or NULL to request the default tables. The result is -always zero. +context. The second argument must point to a set of PCRE2 character tables or +be NULL to request the default tables. The result is always zero. Character +tables can be created by calling \fBpcre2_maketables()\fP or by running the +\fBpcre2_dftables\fP maintenance command in binary mode (see the +.\" HREF +\fBpcre2build\fP +.\" +documentation). .P There is a complete description of the PCRE2 native API in the .\" HREF diff --git a/dist2/doc/pcre2_substitute.3 b/dist2/doc/pcre2_substitute.3 index 7da668cb..cceb7846 100644 --- a/dist2/doc/pcre2_substitute.3 +++ b/dist2/doc/pcre2_substitute.3 @@ -1,4 +1,4 @@ -.TH PCRE2_SUBSTITUTE 3 "04 April 2017" "PCRE2 10.30" +.TH PCRE2_SUBSTITUTE 3 "22 January 2020" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH SYNOPSIS @@ -36,8 +36,8 @@ Its arguments are: \fIoutlengthptr\fP Points to the length of the output buffer .sp A match data block is needed only if you want to inspect the data from the -match that is returned in that block. A match context is needed only if you -want to: +final match that is returned in that block or if PCRE2_SUBSTITUTE_MATCHED is +set. A match context is needed only if you want to: .sp Set up a callout function Set a matching offset limit @@ -45,9 +45,13 @@ want to: Change the backtracking depth limit Set custom memory management in the match context .sp -The \fIlength\fP, \fIstartoffset\fP and \fIrlength\fP values are code -units, not characters, as is the contents of the variable pointed at by -\fIoutlengthptr\fP, which is updated to the actual length of the new string. +The \fIlength\fP, \fIstartoffset\fP and \fIrlength\fP values are code units, +not characters, as is the contents of the variable pointed at by +\fIoutlengthptr\fP. This variable must contain the length of the output buffer +when the function is called. If the function is successful, the value is +changed to the length of the new string, excluding the trailing zero that is +automatically added. +.P The subject and replacement lengths can be given as PCRE2_ZERO_TERMINATED for zero-terminated strings. The options are: .sp @@ -66,12 +70,22 @@ zero-terminated strings. The options are: PCRE2_UTF was set at compile time) PCRE2_SUBSTITUTE_EXTENDED Do extended replacement processing PCRE2_SUBSTITUTE_GLOBAL Replace all occurrences in the subject + PCRE2_SUBSTITUTE_LITERAL The replacement string is literal + PCRE2_SUBSTITUTE_MATCHED Use pre-existing match data for 1st match PCRE2_SUBSTITUTE_OVERFLOW_LENGTH If overflow, compute needed length + PCRE2_SUBSTITUTE_REPLACEMENT_ONLY Return only replacement string(s) PCRE2_SUBSTITUTE_UNKNOWN_UNSET Treat unknown group as unset PCRE2_SUBSTITUTE_UNSET_EMPTY Simple unset insert = empty string .sp +If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_EXTENDED, +PCRE2_SUBSTITUTE_UNKNOWN_UNSET, and PCRE2_SUBSTITUTE_UNSET_EMPTY are ignored. +.P +If PCRE2_SUBSTITUTE_MATCHED is set, \fImatch_data\fP must be non-zero; its +contents must be the result of a call to \fBpcre2_match()\fP using the same +pattern and subject. +.P The function returns the number of substitutions, which may be zero if there -were no matches. The result can be greater than one only when +are no matches. The result may be greater than one only when PCRE2_SUBSTITUTE_GLOBAL is set. In the event of an error, a negative error code is returned. .P diff --git a/dist2/doc/pcre2api.3 b/dist2/doc/pcre2api.3 index d219466c..148dca62 100644 --- a/dist2/doc/pcre2api.3 +++ b/dist2/doc/pcre2api.3 @@ -1,4 +1,4 @@ -.TH PCRE2API 3 "14 February 2019" "PCRE2 10.33" +.TH PCRE2API 3 "04 November 2020" "PCRE2 10.36" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .sp @@ -88,7 +88,7 @@ document for an overview of all the PCRE2 documentation. .B " uint32_t \fIvalue\fP);" .sp .B int pcre2_set_character_tables(pcre2_compile_context *\fIccontext\fP, -.B " const unsigned char *\fItables\fP);" +.B " const uint8_t *\fItables\fP);" .sp .B int pcre2_set_compile_extra_options(pcre2_compile_context *\fIccontext\fP, .B " uint32_t \fIextra_options\fP);" @@ -187,7 +187,7 @@ document for an overview of all the PCRE2 documentation. .B int pcre2_substitute(const pcre2_code *\fIcode\fP, PCRE2_SPTR \fIsubject\fP, .B " PCRE2_SIZE \fIlength\fP, PCRE2_SIZE \fIstartoffset\fP," .B " uint32_t \fIoptions\fP, pcre2_match_data *\fImatch_data\fP," -.B " pcre2_match_context *\fImcontext\fP, PCRE2_SPTR \fIreplacementzfP," +.B " pcre2_match_context *\fImcontext\fP, PCRE2_SPTR \fIreplacementz\fP," .B " PCRE2_SIZE \fIrlength\fP, PCRE2_UCHAR *\fIoutputbuffer\fP," .B " PCRE2_SIZE *\fIoutlengthptr\fP);" .fi @@ -245,7 +245,10 @@ document for an overview of all the PCRE2 documentation. .B int pcre2_get_error_message(int \fIerrorcode\fP, PCRE2_UCHAR *\fIbuffer\fP, .B " PCRE2_SIZE \fIbufflen\fP);" .sp -.B const unsigned char *pcre2_maketables(pcre2_general_context *\fIgcontext\fP); +.B const uint8_t *pcre2_maketables(pcre2_general_context *\fIgcontext\fP); +.sp +.B void pcre2_maketables_free(pcre2_general_context *\fIgcontext\fP, +.B " const uint8_t *\fItables\fP);" .sp .B int pcre2_pattern_info(const pcre2_code *\fIcode\fP, uint32_t \fIwhat\fP, .B " void *\fIwhere\fP);" @@ -561,24 +564,53 @@ documentation for more details. .P In a more complicated situation, where patterns are compiled only when they are first needed, but are still shared between threads, pointers to compiled -patterns must be protected from simultaneous writing by multiple threads, at -least until a pattern has been compiled. The logic can be something like this: +patterns must be protected from simultaneous writing by multiple threads. This +is somewhat tricky to do correctly. If you know that writing to a pointer is +atomic in your environment, you can use logic like this: .sp Get a read-only (shared) lock (mutex) for pointer if (pointer == NULL) { Get a write (unique) lock for pointer - pointer = pcre2_compile(... + if (pointer == NULL) pointer = pcre2_compile(... } Release the lock Use pointer in pcre2_match() .sp Of course, testing for compilation errors should also be included in the code. .P -If JIT is being used, but the JIT compilation is not being done immediately, -(perhaps waiting to see if the pattern is used often enough) similar logic is -required. JIT compilation updates a pointer within the compiled code block, so -a thread must gain unique write access to the pointer before calling +The reason for checking the pointer a second time is as follows: Several +threads may have acquired the shared lock and tested the pointer for being +NULL, but only one of them will be given the write lock, with the rest kept +waiting. The winning thread will compile the pattern and store the result. +After this thread releases the write lock, another thread will get it, and if +it does not retest pointer for being NULL, will recompile the pattern and +overwrite the pointer, creating a memory leak and possibly causing other +issues. +.P +In an environment where writing to a pointer may not be atomic, the above logic +is not sufficient. The thread that is doing the compiling may be descheduled +after writing only part of the pointer, which could cause other threads to use +an invalid value. Instead of checking the pointer itself, a separate "pointer +is valid" flag (that can be updated atomically) must be used: +.sp + Get a read-only (shared) lock (mutex) for pointer + if (!pointer_is_valid) + { + Get a write (unique) lock for pointer + if (!pointer_is_valid) + { + pointer = pcre2_compile(... + pointer_is_valid = TRUE + } + } + Release the lock + Use pointer in pcre2_match() +.sp +If JIT is being used, but the JIT compilation is not being done immediately +(perhaps waiting to see if the pattern is used often enough), similar logic is +required. JIT compilation updates a value within the compiled code block, so a +thread must gain unique write access to the pointer before calling \fBpcre2_jit_compile()\fP. Alternatively, \fBpcre2_code_copy()\fP or \fBpcre2_code_copy_with_tables()\fP can be used to obtain a private copy of the compiled code before calling the JIT compiler. @@ -725,10 +757,10 @@ interpreted matching functions, \fIpcre2_match()\fP and .sp .nf .B int pcre2_set_character_tables(pcre2_compile_context *\fIccontext\fP, -.B " const unsigned char *\fItables\fP);" +.B " const uint8_t *\fItables\fP);" .fi .sp -The value must be the result of a call to \fIpcre2_maketables()\fP, whose only +The value must be the result of a call to \fBpcre2_maketables()\fP, whose only argument is a general context. This function builds a set of character tables in the current locale. .sp @@ -1031,12 +1063,13 @@ less than the limit set by the caller of \fBpcre2_match()\fP or .sp .B int pcre2_config(uint32_t \fIwhat\fP, void *\fIwhere\fP); .P -The function \fBpcre2_config()\fP makes it possible for a PCRE2 client to -discover which optional features have been compiled into the PCRE2 library. The +The function \fBpcre2_config()\fP makes it possible for a PCRE2 client to find +the value of certain configuration parameters and to discover which optional +features have been compiled into the PCRE2 library. The .\" HREF \fBpcre2build\fP .\" -documentation has more details about these optional features. +documentation has more details about these features. .P The first argument for \fBpcre2_config()\fP specifies which information is required. The second argument is a pointer to memory into which the information @@ -1150,6 +1183,16 @@ over compilation stack usage, see \fBpcre2_set_compile_recursion_guard()\fP. This parameter is obsolete and should not be used in new code. The output is a uint32_t integer that is always set to zero. .sp + PCRE2_CONFIG_TABLES_LENGTH +.sp +The output is a uint32_t integer that gives the length of PCRE2's character +processing tables in bytes. For details of these tables see the +.\" HTML <a href="#localesupport"> +.\" </a> +section on locale support +.\" +below. +.sp PCRE2_CONFIG_UNICODE_VERSION .sp The \fIwhere\fP argument should point to a buffer that is at least 24 code @@ -1285,13 +1328,14 @@ and \fBpcre2_compile()\fP returns a non-NULL value. .P There are nearly 100 positive error codes that \fBpcre2_compile()\fP may return if it finds an error in the pattern. There are also some negative error codes -that are used for invalid UTF strings. These are the same as given by -\fBpcre2_match()\fP and \fBpcre2_dfa_match()\fP, and are described in the +that are used for invalid UTF strings when validity checking is in force. These +are the same as given by \fBpcre2_match()\fP and \fBpcre2_dfa_match()\fP, and +are described in the .\" HREF \fBpcre2unicode\fP .\" -page. There is no separate documentation for the positive error codes, because -the textual error messages that are obtained by calling the +documentation. There is no separate documentation for the positive error codes, +because the textual error messages that are obtained by calling the \fBpcre2_get_error_message()\fP function (see "Obtaining a textual error message" .\" HTML <a href="#geterrormessage"> @@ -1416,13 +1460,16 @@ documentation. .sp If this bit is set, letters in the pattern match both upper and lower case letters in the subject. It is equivalent to Perl's /i option, and it can be -changed within a pattern by a (?i) option setting. If PCRE2_UTF is set, Unicode -properties are used for all characters with more than one other case, and for -all characters whose code points are greater than U+007F. For lower valued -characters with only one other case, a lookup table is used for speed. When -PCRE2_UTF is not set, a lookup table is used for all code points less than 256, -and higher code points (available only in 16-bit or 32-bit mode) are treated as -not having another case. +changed within a pattern by a (?i) option setting. If either PCRE2_UTF or +PCRE2_UCP is set, Unicode properties are used for all characters with more than +one other case, and for all characters whose code points are greater than +U+007F. Note that there are two ASCII characters, K and S, that, in addition to +their lower case ASCII equivalents, are case-equivalent with U+212A (Kelvin +sign) and U+017F (long S) respectively. For lower valued characters with only +one other case, a lookup table is used for speed. When neither PCRE2_UTF nor +PCRE2_UCP is set, a lookup table is used for all code points less than 256, and +higher code points (available only in 16-bit or 32-bit mode) are treated as not +having another case. .sp PCRE2_DOLLAR_ENDONLY .sp @@ -1557,10 +1604,20 @@ expression engine is not the most efficient way of doing it. If you are doing a lot of literal matching and are worried about efficiency, you should consider using other approaches. The only other main options that are allowed with PCRE2_LITERAL are: PCRE2_ANCHORED, PCRE2_ENDANCHORED, PCRE2_AUTO_CALLOUT, -PCRE2_CASELESS, PCRE2_FIRSTLINE, PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, -PCRE2_UTF, and PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EXTRA_MATCH_LINE -and PCRE2_EXTRA_MATCH_WORD are also supported. Any other options cause an -error. +PCRE2_CASELESS, PCRE2_FIRSTLINE, PCRE2_MATCH_INVALID_UTF, +PCRE2_NO_START_OPTIMIZE, PCRE2_NO_UTF_CHECK, PCRE2_UTF, and +PCRE2_USE_OFFSET_LIMIT. The extra options PCRE2_EXTRA_MATCH_LINE and +PCRE2_EXTRA_MATCH_WORD are also supported. Any other options cause an error. +.sp + PCRE2_MATCH_INVALID_UTF +.sp +This option forces PCRE2_UTF (see below) and also enables support for matching +by \fBpcre2_match()\fP in subject strings that contain invalid UTF sequences. +This facility is not supported for DFA matching. For details, see the +.\" HREF +\fBpcre2unicode\fP +.\" +documentation. .sp PCRE2_MATCH_UNSET_BACKREF .sp @@ -1690,17 +1747,22 @@ subject string does not happen. The first match attempt is run starting from "D" and when this fails, (*COMMIT) prevents any further matches being tried, so the overall result is "no match". .P -There are also other start-up optimizations. For example, a minimum length for -the subject may be recorded. Consider the pattern +As another start-up optimization makes use of a minimum length for a matching +subject, which is recorded when possible. Consider the pattern .sp - (*MARK:A)(X|Y) + (*MARK:1)B(*MARK:2)(X|Y) .sp -The minimum length for a match is one character. If the subject is "ABC", there -will be attempts to match "ABC", "BC", and "C". An attempt to match an empty -string at the end of the subject does not take place, because PCRE2 knows that -the subject is now too short, and so the (*MARK) is never encountered. In this -case, the optimization does not affect the overall match result, which is still -"no match", but it does affect the auxiliary information that is returned. +The minimum length for a match is two characters. If the subject is "XXBB", the +"starting character" optimization skips "XX", then tries to match "BB", which +is long enough. In the process, (*MARK:2) is encountered and remembered. When +the match attempt fails, the next "B" is found, but there is only one character +left, so there are no more attempts, and "no match" is returned with the "last +mark seen" set to "2". If NO_START_OPTIMIZE is set, however, matches are tried +at every possible starting position, including at the end of the subject, where +(*MARK:1) is encountered, but there is no "B", so the "last mark seen" that is +returned is "1". In this case, the optimizations do not affect the overall +match result, which is still "no match", but they do affect the auxiliary +information that is returned. .sp PCRE2_NO_UTF_CHECK .sp @@ -1750,10 +1812,11 @@ are not representable in UTF-16. .sp PCRE2_UCP .sp -This option changes the way PCRE2 processes \eB, \eb, \eD, \ed, \eS, \es, \eW, -\ew, and some of the POSIX character classes. By default, only ASCII characters -are recognized, but if PCRE2_UCP is set, Unicode properties are used instead to -classify characters. More details are given in the section on +This option has two effects. Firstly, it change the way PCRE2 processes \eB, +\eb, \eD, \ed, \eS, \es, \eW, \ew, and some of the POSIX character classes. By +default, only ASCII characters are recognized, but if PCRE2_UCP is set, Unicode +properties are used instead to classify characters. More details are given in +the section on .\" HTML <a href="pcre2pattern.html#genericchartypes"> .\" </a> generic character types @@ -1763,8 +1826,13 @@ in the \fBpcre2pattern\fP .\" page. If you set PCRE2_UCP, matching one of the items it affects takes much -longer. The option is available only if PCRE2 has been compiled with Unicode -support (which is the default). +longer. +.P +The second effect of PCRE2_UCP is to force the use of Unicode properties for +upper/lower casing operations on characters with code points greater than 127, +even when PCRE2_UTF is not set. This makes it possible, for example, to process +strings in the 16-bit UCS-2 code. This option is available only if PCRE2 has +been compiled with Unicode support (which is the default). .sp PCRE2_UNGREEDY .sp @@ -1927,36 +1995,51 @@ Most (but not all) patterns can be optimized by the JIT compiler. .SH "LOCALE SUPPORT" .rs .sp +.nf +.B const uint8_t *pcre2_maketables(pcre2_general_context *\fIgcontext\fP); +.sp +.B void pcre2_maketables_free(pcre2_general_context *\fIgcontext\fP, +.B " const uint8_t *\fItables\fP);" +.fi +.P PCRE2 handles caseless matching, and determines whether characters are letters, digits, or whatever, by reference to a set of tables, indexed by character code -point. This applies only to characters whose code points are less than 256. By -default, higher-valued code points never match escapes such as \ew or \ed. -However, if PCRE2 is built with Unicode support, all characters can be tested -with \ep and \eP, or, alternatively, the PCRE2_UCP option can be set when a -pattern is compiled; this causes \ew and friends to use Unicode property -support instead of the built-in tables. +point. However, this applies only to characters whose code points are less than +256. By default, higher-valued code points never match escapes such as \ew or +\ed. +.P +When PCRE2 is built with Unicode support (the default), the Unicode properties +of all characters can be tested with \ep and \eP, or, alternatively, the +PCRE2_UCP option can be set when a pattern is compiled; this causes \ew and +friends to use Unicode property support instead of the built-in tables. +PCRE2_UCP also causes upper/lower casing operations on characters with code +points greater than 127 to use Unicode properties. These effects apply even +when PCRE2_UTF is not set. .P The use of locales with Unicode is discouraged. If you are handling characters -with code points greater than 128, you should either use Unicode support, or +with code points greater than 127, you should either use Unicode support, or use locales, but not try to mix the two. .P -PCRE2 contains an internal set of character tables that are used by default. +PCRE2 contains a built-in set of character tables that are used by default. These are sufficient for many applications. Normally, the internal tables recognize only ASCII characters. However, when PCRE2 is built, it is possible to cause the internal tables to be rebuilt in the default "C" locale of the local system, which may cause them to be different. .P -The internal tables can be overridden by tables supplied by the application +The built-in tables can be overridden by tables supplied by the application that calls PCRE2. These may be created in a different locale from the default. As more and more applications change to using Unicode, the need for this locale support is expected to die away. .P External tables are built by calling the \fBpcre2_maketables()\fP function, in -the relevant locale. The result can be passed to \fBpcre2_compile()\fP as often -as necessary, by creating a compile context and calling -\fBpcre2_set_character_tables()\fP to set the tables pointer therein. For -example, to build and use tables that are appropriate for the French locale -(where accented characters with values greater than 128 are treated as +the relevant locale. The only argument to this function is a general context, +which can be used to pass a custom memory allocator. If the argument is NULL, +the system \fBmalloc()\fP is used. The result can be passed to +\fBpcre2_compile()\fP as often as necessary, by creating a compile context and +calling \fBpcre2_set_character_tables()\fP to set the tables pointer therein. +.P +For example, to build and use tables that are appropriate for the French locale +(where accented characters with values greater than 127 are treated as letters), the following code could be used: .sp setlocale(LC_CTYPE, "fr_FR"); @@ -1966,15 +2049,39 @@ letters), the following code could be used: re = pcre2_compile(..., ccontext); .sp The locale name "fr_FR" is used on Linux and other Unix-like systems; if you -are using Windows, the name for the French locale is "french". It is the -caller's responsibility to ensure that the memory containing the tables remains -available for as long as it is needed. +are using Windows, the name for the French locale is "french". .P The pointer that is passed (via the compile context) to \fBpcre2_compile()\fP -is saved with the compiled pattern, and the same tables are used by -\fBpcre2_match()\fP and \fBpcre_dfa_match()\fP. Thus, for any single pattern, -compilation and matching both happen in the same locale, but different patterns -can be processed in different locales. +is saved with the compiled pattern, and the same tables are used by the +matching functions. Thus, for any single pattern, compilation and matching both +happen in the same locale, but different patterns can be processed in different +locales. +.P +It is the caller's responsibility to ensure that the memory containing the +tables remains available while they are still in use. When they are no longer +needed, you can discard them using \fBpcre2_maketables_free()\fP, which should +pass as its first parameter the same global context that was used to create the +tables. +. +. +.SS "Saving locale tables" +.rs +.sp +The tables described above are just a sequence of binary bytes, which makes +them independent of hardware characteristics such as endianness or whether the +processor is 32-bit or 64-bit. A copy of the result of \fBpcre2_maketables()\fP +can therefore be saved in a file or elsewhere and re-used later, even in a +different program or on another computer. The size of the tables (number of +bytes) must be obtained by calling \fBpcre2_config()\fP with the +PCRE2_CONFIG_TABLES_LENGTH option because \fBpcre2_maketables()\fP does not +return this value. Note that the \fBpcre2_dftables\fP program, which is part of +the PCRE2 build system, can be used stand-alone to create a file that contains +a set of binary tables. See the +.\" HTML <a href="pcre2build.html#createtables"> +.\" </a> +\fBpcre2build\fP +.\" +documentation for details. . . .\" HTML <a name="infoaboutpattern"></a> @@ -1987,7 +2094,7 @@ can be processed in different locales. .P The \fBpcre2_pattern_info()\fP function returns general information about a compiled pattern. For information about callouts, see the -.\" HTML <a href="pcre2pattern.html#infoaboutcallouts"> +.\" HTML <a href="#infoaboutcallouts"> .\" </a> next section. .\" @@ -2004,7 +2111,7 @@ the following negative numbers: PCRE2_ERROR_BADOPTION the value of \fIwhat\fP was invalid PCRE2_ERROR_UNSET the requested field is not set .sp -The "magic number" is placed at the start of each compiled pattern as an simple +The "magic number" is placed at the start of each compiled pattern as a simple check against passing an arbitrary memory pointer. Here is a typical call of \fBpcre2_pattern_info()\fP, to obtain the length of the compiled pattern: .sp @@ -2061,7 +2168,7 @@ options returned for PCRE2_INFO_ALLOPTIONS. PCRE2_INFO_BACKREFMAX .sp Return the number of the highest backreference in the pattern. The third -argument should point to an \fBuint32_t\fP variable. Named capture groups +argument should point to a \fBuint32_t\fP variable. Named capture groups acquire numbers as well as names, and these count towards the highest backreference. Backreferences such as \e4 or \eg{12} match the captured characters of the given group, but in addition, the check that a capture @@ -2079,7 +2186,7 @@ that \eR matches only CR, LF, or CRLF. .sp Return the highest capture group number in the pattern. In patterns where (?| is not used, this is also the total number of capture groups. The third -argument should point to an \fBuint32_t\fP variable. +argument should point to a \fBuint32_t\fP variable. .sp PCRE2_INFO_DEPTHLIMIT .sp @@ -2104,7 +2211,7 @@ returned. Otherwise NULL is returned. The third argument should point to a PCRE2_INFO_FIRSTCODETYPE .sp Return information about the first code unit of any matched string, for a -non-anchored pattern. The third argument should point to an \fBuint32_t\fP +non-anchored pattern. The third argument should point to a \fBuint32_t\fP variable. If there is a fixed first value, for example, the letter "c" from a pattern such as (cat|cow|coyote), 1 is returned, and the value can be retrieved using PCRE2_INFO_FIRSTCODEUNIT. If there is no fixed first value, but it is @@ -2116,7 +2223,7 @@ is returned. .sp Return the value of the first code unit of any matched string for a pattern where PCRE2_INFO_FIRSTCODETYPE returns 1; otherwise return 0. The third -argument should point to an \fBuint32_t\fP variable. In the 8-bit library, the +argument should point to a \fBuint32_t\fP variable. In the 8-bit library, the value is always less than 256. In the 16-bit library the value can be up to 0xffff. In the 32-bit library in UTF-32 mode the value can be up to 0x10ffff, and up to 0xffffffff when not using UTF-32 mode. @@ -2132,12 +2239,12 @@ pattern. Each additional capture group adds two PCRE2_SIZE variables. PCRE2_INFO_HASBACKSLASHC .sp Return 1 if the pattern contains any instances of \eC, otherwise 0. The third -argument should point to an \fBuint32_t\fP variable. +argument should point to a \fBuint32_t\fP variable. .sp PCRE2_INFO_HASCRORLF .sp Return 1 if the pattern contains any explicit matches for CR or LF characters, -otherwise 0. The third argument should point to an \fBuint32_t\fP variable. An +otherwise 0. The third argument should point to a \fBuint32_t\fP variable. An explicit match is either a literal CR or LF character, or \er or \en or one of the equivalent hexadecimal or octal escape sequences. .sp @@ -2153,7 +2260,7 @@ defaulted by the caller of the match function. PCRE2_INFO_JCHANGED .sp Return 1 if the (?J) or (?-J) option setting is used in the pattern, otherwise -0. The third argument should point to an \fBuint32_t\fP variable. (?J) and +0. The third argument should point to a \fBuint32_t\fP variable. (?J) and (?-J) set and unset the local PCRE2_DUPNAMES option, respectively. .sp PCRE2_INFO_JITSIZE @@ -2165,7 +2272,7 @@ return zero. The third argument should point to a \fBsize_t\fP variable. PCRE2_INFO_LASTCODETYPE .sp Returns 1 if there is a rightmost literal code unit that must exist in any -matched string, other than at its start. The third argument should point to an +matched string, other than at its start. The third argument should point to a \fBuint32_t\fP variable. If there is no such value, 0 is returned. When 1 is returned, the code unit value itself can be retrieved using PCRE2_INFO_LASTCODEUNIT. For anchored patterns, a last literal value is @@ -2178,12 +2285,12 @@ PCRE2_INFO_LASTCODEUNIT), but for /^a\edz\ed/ the returned value is 0. Return the value of the rightmost literal code unit that must exist in any matched string, other than at its start, for a pattern where PCRE2_INFO_LASTCODETYPE returns 1. Otherwise, return 0. The third argument -should point to an \fBuint32_t\fP variable. +should point to a \fBuint32_t\fP variable. .sp PCRE2_INFO_MATCHEMPTY .sp Return 1 if the pattern might match an empty string, otherwise 0. The third -argument should point to an \fBuint32_t\fP variable. When a pattern contains +argument should point to a \fBuint32_t\fP variable. When a pattern contains recursive subroutine calls it is not always possible to determine whether or not it can match an empty string. PCRE2 takes a cautious approach and returns 1 in such cases. @@ -2199,26 +2306,35 @@ defaulted by the caller of the match function. .sp PCRE2_INFO_MAXLOOKBEHIND .sp -Return the number of characters (not code units) in the longest lookbehind -assertion in the pattern. The third argument should point to a uint32_t -integer. This information is useful when doing multi-segment matching using the -partial matching facilities. Note that the simple assertions \eb and \eB -require a one-character lookbehind. \eA also registers a one-character -lookbehind, though it does not actually inspect the previous character. This is -to ensure that at least one character from the old segment is retained when a -new segment is processed. Otherwise, if there are no lookbehinds in the -pattern, \eA might match incorrectly at the start of a second or subsequent -segment. +A lookbehind assertion moves back a certain number of characters (not code +units) when it starts to process each of its branches. This request returns the +largest of these backward moves. The third argument should point to a uint32_t +integer. The simple assertions \eb and \eB require a one-character lookbehind +and cause PCRE2_INFO_MAXLOOKBEHIND to return 1 in the absence of anything +longer. \eA also registers a one-character lookbehind, though it does not +actually inspect the previous character. +.P +Note that this information is useful for multi-segment matching only +if the pattern contains no nested lookbehinds. For example, the pattern +(?<=a(?<=ba)c) returns a maximum lookbehind of 2, but when it is processed, the +first lookbehind moves back by two characters, matches one character, then the +nested lookbehind also moves back by two characters. This puts the matching +point three characters earlier than it was at the start. +PCRE2_INFO_MAXLOOKBEHIND is really only useful as a debugging tool. See the +.\" HREF +\fBpcre2partial\fP +.\" +documentation for a discussion of multi-segment matching. .sp PCRE2_INFO_MINLENGTH .sp If a minimum length for matching subject strings was computed, its value is -returned. Otherwise the returned value is 0. The value is a number of -characters, which in UTF mode may be different from the number of code units. -The third argument should point to an \fBuint32_t\fP variable. The value is a -lower bound to the length of any matching string. There may not be any strings -of that length that do actually match, but every string that does match is at -least that long. +returned. Otherwise the returned value is 0. This value is not computed when +PCRE2_NO_START_OPTIMIZE is set. The value is a number of characters, which in +UTF mode may be different from the number of code units. The third argument +should point to a \fBuint32_t\fP variable. The value is a lower bound to the +length of any matching string. There may not be any strings of that length that +do actually match, but every string that does match is at least that long. .sp PCRE2_INFO_NAMECOUNT PCRE2_INFO_NAMEENTRYSIZE @@ -2635,15 +2751,23 @@ of JIT; it forces matching to be done by the interpreter. PCRE2_NO_UTF_CHECK .sp When PCRE2_UTF is set at compile time, the validity of the subject as a UTF -string is checked by default when \fBpcre2_match()\fP is subsequently called. -If a non-zero starting offset is given, the check is applied only to that part -of the subject that could be inspected during matching, and there is a check -that the starting offset points to the first code unit of a character or to the -end of the subject. If there are no lookbehind assertions in the pattern, the -check starts at the starting offset. Otherwise, it starts at the length of the -longest lookbehind before the starting offset, or at the start of the subject -if there are not that many characters before the starting offset. Note that the -sequences \eb and \eB are one-character lookbehinds. +string is checked unless PCRE2_NO_UTF_CHECK is passed to \fBpcre2_match()\fP or +PCRE2_MATCH_INVALID_UTF was passed to \fBpcre2_compile()\fP. The latter special +case is discussed in detail in the +.\" HREF +\fBpcre2unicode\fP +.\" +documentation. +.P +In the default case, if a non-zero starting offset is given, the check is +applied only to that part of the subject that could be inspected during +matching, and there is a check that the starting offset points to the first +code unit of a character or to the end of the subject. If there are no +lookbehind assertions in the pattern, the check starts at the starting offset. +Otherwise, it starts at the length of the longest lookbehind before the +starting offset, or at the start of the subject if there are not that many +characters before the starting offset. Note that the sequences \eb and \eB are +one-character lookbehinds. .P The check is carried out before any other processing takes place, and a negative error code is returned if the check fails. There are several UTF error @@ -2666,29 +2790,34 @@ in the .\" HREF \fBpcre2unicode\fP .\" -page. +documentation. .P -If you know that your subject is valid, and you want to skip these checks for +If you know that your subject is valid, and you want to skip this check for performance reasons, you can set the PCRE2_NO_UTF_CHECK option when calling \fBpcre2_match()\fP. You might want to do this for the second and subsequent -calls to \fBpcre2_match()\fP if you are making repeated calls to find other +calls to \fBpcre2_match()\fP if you are making repeated calls to find multiple matches in the same subject string. .P -\fBWarning:\fP When PCRE2_NO_UTF_CHECK is set, the effect of passing an invalid +\fBWarning:\fP Unless PCRE2_MATCH_INVALID_UTF was set at compile time, when +PCRE2_NO_UTF_CHECK is set at match time the effect of passing an invalid string as a subject, or an invalid value of \fIstartoffset\fP, is undefined. -Your program may crash or loop indefinitely. +Your program may crash or loop indefinitely or give wrong results. .sp PCRE2_PARTIAL_HARD PCRE2_PARTIAL_SOFT .sp These options turn on the partial matching feature. A partial match occurs if the end of the subject string is reached successfully, but there are not enough -subject characters to complete the match. If this happens when -PCRE2_PARTIAL_SOFT (but not PCRE2_PARTIAL_HARD) is set, matching continues by -testing any remaining alternatives. Only if no complete match can be found is -PCRE2_ERROR_PARTIAL returned instead of PCRE2_ERROR_NOMATCH. In other words, -PCRE2_PARTIAL_SOFT specifies that the caller is prepared to handle a partial -match, but only if no complete match can be found. +subject characters to complete the match. In addition, either at least one +character must have been inspected or the pattern must contain a lookbehind, or +the pattern must be one that could match an empty string. +.P +If this situation arises when PCRE2_PARTIAL_SOFT (but not PCRE2_PARTIAL_HARD) +is set, matching continues by testing any remaining alternatives. Only if no +complete match can be found is PCRE2_ERROR_PARTIAL returned instead of +PCRE2_ERROR_NOMATCH. In other words, PCRE2_PARTIAL_SOFT specifies that the +caller is prepared to handle a partial match, but only if no complete match can +be found. .P If PCRE2_PARTIAL_HARD is set, it overrides PCRE2_PARTIAL_SOFT. In this case, if a partial match is found, \fBpcre2_match()\fP immediately returns @@ -3266,12 +3395,19 @@ same number causes an error at compile time. .B " PCRE2_SIZE *\fIoutlengthptr\fP);" .fi .P -This function calls \fBpcre2_match()\fP and then makes a copy of the subject -string in \fIoutputbuffer\fP, replacing one or more parts that were matched -with the \fIreplacement\fP string, whose length is supplied in \fBrlength\fP. -This can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. -The default is to perform just one replacement, but there is an option that -requests multiple replacements (see PCRE2_SUBSTITUTE_GLOBAL below for details). +This function optionally calls \fBpcre2_match()\fP and then makes a copy of the +subject string in \fIoutputbuffer\fP, replacing parts that were matched with +the \fIreplacement\fP string, whose length is supplied in \fBrlength\fP. This +can be given as PCRE2_ZERO_TERMINATED for a zero-terminated string. There is an +option (see PCRE2_SUBSTITUTE_REPLACEMENT_ONLY below) to return just the +replacement string(s). The default action is to perform just one replacement if +the pattern matches, but there is an option that requests multiple replacements +(see PCRE2_SUBSTITUTE_GLOBAL below). +.P +If successful, \fBpcre2_substitute()\fP returns the number of substitutions +that were carried out. This may be zero if no match was found, and is never +greater than one unless PCRE2_SUBSTITUTE_GLOBAL is set. A negative value is +returned if an error is detected. .P Matches in which a \eK item in a lookahead in the pattern causes the match to end before it starts are not supported, and give rise to an error return. For @@ -3286,32 +3422,79 @@ data block is obtained and freed within this function, using memory management functions from the match context, if provided, or else those that were used to allocate memory for the compiled code. .P -If an external \fImatch_data\fP block is provided, its contents afterwards -are those set by the final call to \fBpcre2_match()\fP. For global changes, -this will have ended in a matching error. The contents of the ovector within -the match data block may or may not have been changed. +If \fImatch_data\fP is not NULL and PCRE2_SUBSTITUTE_MATCHED is not set, the +provided block is used for all calls to \fBpcre2_match()\fP, and its contents +afterwards are the result of the final call. For global changes, this will +always be a no-match error. The contents of the ovector within the match data +block may or may not have been changed. +.P +As well as the usual options for \fBpcre2_match()\fP, a number of additional +options can be set in the \fIoptions\fP argument of \fBpcre2_substitute()\fP. +One such option is PCRE2_SUBSTITUTE_MATCHED. When this is set, an external +\fImatch_data\fP block must be provided, and it must have been used for an +external call to \fBpcre2_match()\fP. The data in the \fImatch_data\fP block +(return code, offset vector) is used for the first substitution instead of +calling \fBpcre2_match()\fP from within \fBpcre2_substitute()\fP. This allows +an application to check for a match before choosing to substitute, without +having to repeat the match. +.P +The contents of the externally supplied match data block are not changed when +PCRE2_SUBSTITUTE_MATCHED is set. If PCRE2_SUBSTITUTE_GLOBAL is also set, +\fBpcre2_match()\fP is called after the first substitution to check for further +matches, but this is done using an internally obtained match data block, thus +always leaving the external block unchanged. +.P +The \fIcode\fP argument is not used for matching before the first substitution +when PCRE2_SUBSTITUTE_MATCHED is set, but it must be provided, even when +PCRE2_SUBSTITUTE_GLOBAL is not set, because it contains information such as the +UTF setting and the number of capturing parentheses in the pattern. +.P +The default action of \fBpcre2_substitute()\fP is to return a copy of the +subject string with matched substrings replaced. However, if +PCRE2_SUBSTITUTE_REPLACEMENT_ONLY is set, only the replacement substrings are +returned. In the global case, multiple replacements are concatenated in the +output buffer. Substitution callouts (see +.\" HTML <a href="#subcallouts"> +.\" </a> +below) +.\" +can be used to separate them if necessary. .P -The \fIoutlengthptr\fP argument must point to a variable that contains the -length, in code units, of the output buffer. If the function is successful, the -value is updated to contain the length of the new string, excluding the -trailing zero that is automatically added. +The \fIoutlengthptr\fP argument of \fBpcre2_substitute()\fP must point to a +variable that contains the length, in code units, of the output buffer. If the +function is successful, the value is updated to contain the length in code +units of the new string, excluding the trailing zero that is automatically +added. .P If the function is not successful, the value set via \fIoutlengthptr\fP depends on the type of error. For syntax errors in the replacement string, the value is the offset in the replacement string where the error was detected. For other errors, the value is PCRE2_UNSET by default. This includes the case of the -output buffer being too small, unless PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set -(see below), in which case the value is the minimum length needed, including -space for the trailing zero. Note that in order to compute the required length, -\fBpcre2_substitute()\fP has to simulate all the matching and copying, instead -of giving an error return as soon as the buffer overflows. Note also that the -length is in code units, not bytes. -.P -In the replacement string, which is interpreted as a UTF string in UTF mode, -and is checked for UTF validity unless the PCRE2_NO_UTF_CHECK option is set, a -dollar character is an escape character that can specify the insertion of -characters from capture groups or names from (*MARK) or other control verbs -in the pattern. The following forms are always recognized: +output buffer being too small, unless PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set. +.P +PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output buffer is +too small. The default action is to return PCRE2_ERROR_NOMEMORY immediately. If +this option is set, however, \fBpcre2_substitute()\fP continues to go through +the motions of matching and substituting (without, of course, writing anything) +in order to compute the size of buffer that is needed. This value is passed +back via the \fIoutlengthptr\fP variable, with the result of the function still +being PCRE2_ERROR_NOMEMORY. +.P +Passing a buffer size of zero is a permitted way of finding out how much memory +is needed for given substitution. However, this does mean that the entire +operation is carried out twice. Depending on the application, it may be more +efficient to allocate a large buffer and free the excess afterwards, instead of +using PCRE2_SUBSTITUTE_OVERFLOW_LENGTH. +.P +The replacement string, which is interpreted as a UTF string in UTF mode, is +checked for UTF validity unless PCRE2_NO_UTF_CHECK is set. An invalid UTF +replacement string causes an immediate return with the relevant UTF error code. +.P +If PCRE2_SUBSTITUTE_LITERAL is set, the replacement string is not interpreted +in any way. By default, however, a dollar character is an escape character that +can specify the insertion of characters from capture groups and names from +(*MARK) or other control verbs in the pattern. The following forms are always +recognized: .sp $$ insert a dollar character $<n> or ${<n>} insert the contents of group <n> @@ -3334,9 +3517,6 @@ facility can be used to perform simple simultaneous substitutions, as this apple lemon 2: pear orange .sp -As well as the usual options for \fBpcre2_match()\fP, a number of additional -options can be set in the \fIoptions\fP argument of \fBpcre2_substitute()\fP. -.P PCRE2_SUBSTITUTE_GLOBAL causes the function to iterate over the subject string, replacing every matching substring. If this option is not set, only the first matching substring is replaced. The search for matches takes place in the @@ -3347,7 +3527,7 @@ set in the match context, searching stops when that limit is reached. .P You can restrict the effect of a global substitution to a portion of the subject string by setting either or both of \fIstartoffset\fP and an offset -limit. Here is a \fPpcre2test\fP example: +limit. Here is a \fBpcre2test\fP example: .sp /B/g,replace=!,use_offset_limit ABC ABC ABC ABC\e=offset=3,offset_limit=12 @@ -3359,20 +3539,6 @@ If this is not successful, the offset is advanced by one character except when CRLF is a valid newline sequence and the next two characters are CR, LF. In this case, the offset is advanced by two characters. .P -PCRE2_SUBSTITUTE_OVERFLOW_LENGTH changes what happens when the output buffer is -too small. The default action is to return PCRE2_ERROR_NOMEMORY immediately. If -this option is set, however, \fBpcre2_substitute()\fP continues to go through -the motions of matching and substituting (without, of course, writing anything) -in order to compute the size of buffer that is needed. This value is passed -back via the \fIoutlengthptr\fP variable, with the result of the function still -being PCRE2_ERROR_NOMEMORY. -.P -Passing a buffer size of zero is a permitted way of finding out how much memory -is needed for given substitution. However, this does mean that the entire -operation is carried out twice. Depending on the application, it may be more -efficient to allocate a large buffer and free the excess afterwards, instead of -using PCRE2_SUBSTITUTE_OVERFLOW_LENGTH. -.P PCRE2_SUBSTITUTE_UNKNOWN_UNSET causes references to capture groups that do not appear in the pattern to be treated as unset groups. This option should be used with care, because it means that a typo in a group name or number no @@ -3403,12 +3569,15 @@ terminating a \eQ quoted sequence) reverts to no case forcing. The sequences \eu and \el force the next character (if it is a letter) to upper or lower case, respectively, and then the state automatically reverts to no case forcing. Case forcing applies to all inserted characters, including those from -capture groups and letters within \eQ...\eE quoted sequences. +capture groups and letters within \eQ...\eE quoted sequences. If either +PCRE2_UTF or PCRE2_UCP was set when the pattern was compiled, Unicode +properties are used for case forcing characters whose code points are greater +than 127. .P Note that case forcing sequences such as \eU...\eE do not nest. For example, the result of processing "\eUaa\eLBB\eEcc\eE" is "AAbbcc"; the final \eE has no effect. Note also that the PCRE2_ALT_BSUX and PCRE2_EXTRA_ALT_BSUX options do -not apply to not apply to replacement strings. +not apply to replacement strings. .P The second effect of setting PCRE2_SUBSTITUTE_EXTENDED is to add more flexibility to capture group substitution. The syntax is similar to that used @@ -3439,13 +3608,17 @@ The PCRE2_SUBSTITUTE_UNSET_EMPTY option does not affect these extended substitutions. However, PCRE2_SUBSTITUTE_UNKNOWN_UNSET does cause unknown groups in the extended syntax forms to be treated as unset. .P -If successful, \fBpcre2_substitute()\fP returns the number of successful -matches. This may be zero if no matches were found, and is never greater than 1 -unless PCRE2_SUBSTITUTE_GLOBAL is set. -.P -In the event of an error, a negative error code is returned. Except for -PCRE2_ERROR_NOMATCH (which is never returned), errors from \fBpcre2_match()\fP -are passed straight back. +If PCRE2_SUBSTITUTE_LITERAL is set, PCRE2_SUBSTITUTE_UNKNOWN_UNSET, +PCRE2_SUBSTITUTE_UNSET_EMPTY, and PCRE2_SUBSTITUTE_EXTENDED are irrelevant and +are ignored. +. +. +.SS "Substitution errors" +.rs +.sp +In the event of an error, \fBpcre2_substitute()\fP returns a negative error +code. Except for PCRE2_ERROR_NOMATCH (which is never returned), errors from +\fBpcre2_match()\fP are passed straight back. .P PCRE2_ERROR_NOSUBSTRING is returned for a non-existent substring insertion, unless PCRE2_SUBSTITUTE_UNKNOWN_UNSET is set. @@ -3459,6 +3632,9 @@ PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set, the size of buffer that is needed is returned via \fIoutlengthptr\fP. Note that this does not happen by default. .P +PCRE2_ERROR_NULL is returned if PCRE2_SUBSTITUTE_MATCHED is set but the +\fImatch_data\fP argument is NULL. +.P PCRE2_ERROR_BADREPLACEMENT is used for miscellaneous syntax errors in the replacement string, with more particular errors being PCRE2_ERROR_BADREPESCAPE (invalid escape sequence), PCRE2_ERROR_REPMISSINGBRACE (closing curly bracket @@ -3476,6 +3652,7 @@ above). .\" . . +.\" HTML <a name="subcallouts"></a> .SS "Substitution callouts" .rs .sp @@ -3775,6 +3952,12 @@ This return is given if \fBpcre2_dfa_match()\fP encounters a condition item that uses a backreference for the condition, or a test for recursion in a specific capture group. These are not supported. .sp + PCRE2_ERROR_DFA_UINVALID_UTF +.sp +This return is given if \fBpcre2_dfa_match()\fP is called for a pattern that +was compiled with PCRE2_MATCH_INVALID_UTF. This is not supported for DFA +matching. +.sp PCRE2_ERROR_DFA_WSSIZE .sp This return is given if \fBpcre2_dfa_match()\fP runs out of space in the @@ -3817,6 +4000,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 14 February 2019 -Copyright (c) 1997-2019 University of Cambridge. +Last updated: 04 November 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/doc/pcre2build.3 b/dist2/doc/pcre2build.3 index f1d28f8b..edea2223 100644 --- a/dist2/doc/pcre2build.3 +++ b/dist2/doc/pcre2build.3 @@ -1,4 +1,4 @@ -.TH PCRE2BUILD 3 "03 March 2019" "PCRE2 10.33" +.TH PCRE2BUILD 3 "20 March 2020" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) . @@ -110,7 +110,7 @@ To build it without Unicode support, add --disable-unicode .sp to the \fBconfigure\fP command. This setting applies to all three libraries. It -is not possible to build one library with Unicode support, and another without, +is not possible to build one library with Unicode support and another without in the same configuration. .P Of itself, Unicode support does not make PCRE2 treat strings as UTF-8, UTF-16 @@ -175,11 +175,11 @@ SELinux. This has no effect if JIT is not enabled. See the \fBpcre2jit\fP .\" documentation for a discussion of JIT usage. When JIT support is enabled, -pcre2grep automatically makes use of it, unless you add +\fBpcre2grep\fP automatically makes use of it, unless you add .sp --disable-pcre2grep-jit .sp -to the "configure" command. +to the \fBconfigure\fP command. . . .SH "NEWLINE RECOGNITION" @@ -317,6 +317,7 @@ used for lookaround assertions, atomic groups, and recursion within patterns. The limit does not apply to JIT matching. . . +.\" HTML <a name="createtables"></a> .SH "CREATING CHARACTER TABLES AT BUILD TIME" .rs .sp @@ -328,12 +329,33 @@ only. If you add --enable-rebuild-chartables .sp to the \fBconfigure\fP command, the distributed tables are no longer used. -Instead, a program called \fBdftables\fP is compiled and run. This outputs the -source for new set of tables, created in the default locale of your C run-time -system. This method of replacing the tables does not work if you are cross -compiling, because \fBdftables\fP is run on the local host. If you need to -create alternative tables when cross compiling, you will have to do so "by -hand". +Instead, a program called \fBpcre2_dftables\fP is compiled and run. This +outputs the source for new set of tables, created in the default locale of your +C run-time system. This method of replacing the tables does not work if you are +cross compiling, because \fBpcre2_dftables\fP needs to be run on the local +host and therefore not compiled with the cross compiler. +.P +If you need to create alternative tables when cross compiling, you will have to +do so "by hand". There may also be other reasons for creating tables manually. +To cause \fBpcre2_dftables\fP to be built on the local host, run a normal +compiling command, and then run the program with the output file as its +argument, for example: +.sp + cc src/pcre2_dftables.c -o pcre2_dftables + ./pcre2_dftables src/pcre2_chartables.c +.sp +This builds the tables in the default locale of the local host. If you want to +specify a locale, you must use the -L option: +.sp + LC_ALL=fr_FR ./pcre2_dftables -L src/pcre2_chartables.c +.sp +You can also specify -b (with or without -L). This causes the tables to be +written in binary instead of as source code. A set of binary tables can be +loaded into memory by an application and passed to \fBpcre2_compile()\fP in the +same way as tables created by calling \fBpcre2_maketables()\fP. The tables are +just a string of bytes, independent of hardware characteristics such as +endianness. This means they can be bundled with an application that runs in +different environments, to ensure consistent behaviour. . . .SH "USING EBCDIC CODE" @@ -417,7 +439,7 @@ default parameter values by adding, for example, --with-pcre2grep-bufsize=51200 --with-pcre2grep-max-bufsize=2097152 .sp -to the \fBconfigure\fP command. The caller of \fPpcre2grep\fP can override +to the \fBconfigure\fP command. The caller of \fBpcre2grep\fP can override these values by using --buffer-size and --max-buffer-size on the command line. . . @@ -548,7 +570,7 @@ support these modifiers. If .sp --disable-percent-zt .sp -is specified, no use is made of the z or t modifiers. Instead or %td or %zu, +is specified, no use is made of the z or t modifiers. Instead of %td or %zu, %lu is used, with a cast for size_t values. . . @@ -610,6 +632,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 03 March 2019 -Copyright (c) 1997-2019 University of Cambridge. +Last updated: 20 March 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/doc/pcre2compat.3 b/dist2/doc/pcre2compat.3 index 39ccc2ea..026e6648 100644 --- a/dist2/doc/pcre2compat.3 +++ b/dist2/doc/pcre2compat.3 @@ -1,13 +1,13 @@ -.TH PCRE2COMPAT 3 "12 February 2019" "PCRE2 10.33" +.TH PCRE2COMPAT 3 "06 October 2020" "PCRE2 10.36" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH "DIFFERENCES BETWEEN PCRE2 AND PERL" .rs .sp -This document describes the differences in the ways that PCRE2 and Perl handle -regular expressions. The differences described here are with respect to Perl -versions 5.26, but as both Perl and PCRE2 are continually changing, the -information may sometimes be out of date. +This document describes some of the differences in the ways that PCRE2 and Perl +handle regular expressions. The differences described here are with respect to +Perl version 5.32.0, but as both Perl and PCRE2 are continually changing, the +information may at times be out of date. .P 1. PCRE2 has only a subset of Perl's Unicode support. Details of what it does have are given in the @@ -21,11 +21,14 @@ they do not mean what you might think. For example, (?!a){3} does not assert that the next three characters are not "a". It just asserts that the next character is not "a" three times (in principle; PCRE2 optimizes this to run the assertion just once). Perl allows some repeat quantifiers on other assertions, -for example, \eb* (but not \eb{3}), but these do not seem to have any use. +for example, \eb* (but not \eb{3}, though oddly it does allow ^{3}), but these +do not seem to have any use. PCRE2 does not allow any kind of quantifier on +non-lookaround assertions. .P 3. Capture groups that occur inside negative lookaround assertions are counted, but their entries in the offsets vector are set only when a negative assertion is a condition that has a matching branch (that is, the condition is false). +Perl may set such capture groups in other circumstances. .P 4. The following Perl escape sequences are not supported: \eF, \el, \eL, \eu, \eU, and \eN when followed by a character name. \eN on its own, matching a @@ -41,10 +44,14 @@ interprets them. built with Unicode support (the default). The properties that can be tested with \ep and \eP are limited to the general category properties such as Lu and Nd, script names such as Greek or Han, and the derived properties Any and L&. -PCRE2 does support the Cs (surrogate) property, which Perl does not; the Perl -documentation says "Because Perl hides the need for the user to understand the -internal representation of Unicode characters, there is no need to implement -the somewhat messy concept of surrogates." +Both PCRE2 and Perl support the Cs (surrogate) property, but in PCRE2 its use +is limited. See the +.\" HREF +\fBpcre2pattern\fP +.\" +documentation for details. The long synonyms for property names that Perl +supports (such as \ep{Letter}) are not supported by PCRE2, nor is it permitted +to prefix any of these properties with "Is". .P 6. PCRE2 supports the \eQ...\eE escape for quoting substrings. Characters in between are treated as literals. However, this is slightly different from @@ -65,7 +72,8 @@ other character. Note the following examples: \eQA\eB\eE A\eB A\eB \eQ\e\eE \e \e\eE .sp -The \eQ...\eE sequence is recognized both inside and outside character classes. +The \eQ...\eE sequence is recognized both inside and outside character classes +by both PCRE2 and Perl. .P 7. Fairly obviously, PCRE2 does not support the (?{code}) and (??{code}) constructions. However, PCRE2 does have a "callout" feature, which allows an @@ -79,13 +87,13 @@ documentation for details. to PCRE2 release 10.23, but from release 10.30 this changed, and backtracking into subroutine calls is now supported, as in Perl. .P -9. If any of the backtracking control verbs are used in a group that is called -as a subroutine (whether or not recursively), their effect is confined to that -group; it does not extend to the surrounding pattern. This is not always the -case in Perl. In particular, if (*THEN) is present in a group that is called as -a subroutine, its action is limited to that group, even if the group does not -contain any | characters. Note that such groups are processed as anchored -at the point where they are tested. +9. In PCRE2, if any of the backtracking control verbs are used in a group that +is called as a subroutine (whether or not recursively), their effect is +confined to that group; it does not extend to the surrounding pattern. This is +not always the case in Perl. In particular, if (*THEN) is present in a group +that is called as a subroutine, its action is limited to that group, even if +the group does not contain any | characters. Note that such groups are +processed as anchored at the point where they are tested. .P 10. If a pattern contains more than one backtracking control verb, the first one that is backtracked onto acts. For example, in the pattern @@ -93,48 +101,49 @@ A(*COMMIT)B(*PRUNE)C a failure in B triggers (*COMMIT), but a failure in C triggers (*PRUNE). Perl's behaviour is more complex; in many cases it is the same as PCRE2, but there are cases where it differs. .P -11. Most backtracking verbs in assertions have their normal actions. They are -not confined to the assertion. -.P -12. There are some differences that are concerned with the settings of captured +11. There are some differences that are concerned with the settings of captured strings when part of a pattern is repeated. For example, matching "aba" against the pattern /^(a(b)?)+$/ in Perl leaves $2 unset, but in PCRE2 it is set to "b". .P -13. PCRE2's handling of duplicate capture group numbers and names is not as +12. PCRE2's handling of duplicate capture group numbers and names is not as general as Perl's. This is a consequence of the fact the PCRE2 works internally just with numbers, using an external table to translate between numbers and -names. In particular, a pattern such as (?|(?<a>A)|(?<b>B), where the two +names. In particular, a pattern such as (?|(?<a>A)|(?<b>B)), where the two capture groups have the same number but different names, is not supported, and causes an error at compile time. If it were allowed, it would not be possible to distinguish which group matched, because both names map to capture group number 1. To avoid this confusing situation, an error is given at compile time. .P -14. Perl used to recognize comments in some places that PCRE2 does not, for +13. Perl used to recognize comments in some places that PCRE2 does not, for example, between the ( and ? at the start of a group. If the /x modifier is set, Perl allowed white space between ( and ? though the latest Perls give an error (for a while it was just deprecated). There may still be some cases where Perl behaves differently. .P -15. Perl, when in warning mode, gives warnings for character classes such as +14. Perl, when in warning mode, gives warnings for character classes such as [A-\ed] or [a-[:digit:]]. It then treats the hyphens as literals. PCRE2 has no warning features, so it gives an error in these cases because they are almost certainly user mistakes. .P -16. In PCRE2, the upper/lower case character properties Lu and Ll are not +15. In PCRE2, the upper/lower case character properties Lu and Ll are not affected when case-independent matching is specified. For example, \ep{Lu} always matches an upper case letter. I think Perl has changed in this respect; -in the release at the time of writing (5.24), \ep{Lu} and \ep{Ll} match all +in the release at the time of writing (5.32), \ep{Lu} and \ep{Ll} match all letters, regardless of case, when case independence is specified. .P +16. From release 5.32.0, Perl locks out the use of \eK in lookaround +assertions. In PCRE2, \eK is acted on when it occurs in positive assertions, +but is ignored in negative assertions. +.P 17. PCRE2 provides some extensions to the Perl regular expression facilities. -Perl 5.10 includes new features that are not in earlier versions of Perl, some +Perl 5.10 included new features that were not in earlier versions of Perl, some of which (such as named parentheses) were in PCRE2 for some time before. This -list is with respect to Perl 5.26: +list is with respect to Perl 5.32: .sp (a) Although lookbehind assertions in PCRE2 must match fixed length strings, -each alternative branch of a lookbehind assertion can match a different length -of string. Perl requires them all to have the same length. +each alternative toplevel branch of a lookbehind assertion can match a +different length of string. Perl requires them all to have the same length. .sp (b) From PCRE2 10.23, backreferences to groups of fixed length are supported in lookbehinds, provided that there is no possibility of referencing a @@ -168,8 +177,12 @@ variable interpolation, but not general hooks on every match. different way and is not Perl-compatible. .sp (l) PCRE2 recognizes some special sequences such as (*CR) or (*NO_JIT) at -the start of a pattern that set overall options that cannot be changed within +the start of a pattern. These set overall options that cannot be changed within the pattern. +.sp +(m) PCRE2 supports non-atomic positive lookaround assertions. This is an +extension to the lookaround facilities. The default, Perl-compatible +lookarounds are atomic. .P 18. The Perl /a modifier restricts /d numbers to pure ascii, and the /aa modifier restricts /i case-insensitive matching to pure ascii, ignoring Unicode @@ -199,6 +212,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 12 February 2019 +Last updated: 06 October 2020 Copyright (c) 1997-2019 University of Cambridge. .fi diff --git a/dist2/doc/pcre2demo.3 b/dist2/doc/pcre2demo.3 index a9e58e26..0d301459 100644 --- a/dist2/doc/pcre2demo.3 +++ b/dist2/doc/pcre2demo.3 @@ -104,12 +104,11 @@ uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE subject_length; -size_t subject_length; pcre2_match_data *match_data; - /************************************************************************** * First, sort out the command line. There is only one possible option at * * the moment, "-g" to request repeated matching to find all occurrences, * @@ -138,12 +137,14 @@ if (argc - i != 2) return 1; } -/* As pattern and subject are char arguments, they can be straightforwardly -cast to PCRE2_SPTR as we are working in 8-bit code units. */ +/* Pattern and subject are char arguments, so they can be straightforwardly +cast to PCRE2_SPTR because we are working in 8-bit code units. The subject +length is cast to PCRE2_SIZE for completeness, though PCRE2_SIZE is in fact +defined to be size_t. */ pattern = (PCRE2_SPTR)argv[i]; subject = (PCRE2_SPTR)argv[i+1]; -subject_length = strlen((char *)subject); +subject_length = (PCRE2_SIZE)strlen((char *)subject); /************************************************************************* @@ -172,17 +173,22 @@ if (re == NULL) /************************************************************************* -* If the compilation succeeded, we call PCRE again, in order to do a * +* If the compilation succeeded, we call PCRE2 again, in order to do a * * pattern match against the subject string. This does just ONE match. If * * further matching is needed, it will be done below. Before running the * -* match we must set up a match_data block for holding the result. * +* match we must set up a match_data block for holding the result. Using * +* pcre2_match_data_create_from_pattern() ensures that the block is * +* exactly the right size for the number of capturing parentheses in the * +* pattern. If you need to know the actual size of a match_data block as * +* a number of bytes, you can find it like this: * +* * +* PCRE2_SIZE match_data_size = pcre2_get_match_data_size(match_data); * *************************************************************************/ -/* Using this function ensures that the block is exactly the right size for -the number of capturing parentheses in the pattern. */ - match_data = pcre2_match_data_create_from_pattern(re, NULL); +/* Now run the match. */ + rc = pcre2_match( re, /* the compiled pattern */ subject, /* the subject string */ @@ -205,7 +211,7 @@ if (rc < 0) default: printf("Matching error %d\en", rc); break; } pcre2_match_data_free(match_data); /* Release memory used for the match */ - pcre2_code_free(re); /* data and the compiled pattern. */ + pcre2_code_free(re); /* data and the compiled pattern. */ return 1; } @@ -249,7 +255,7 @@ application you might want to do things other than print them. */ for (i = 0; i < rc; i++) { PCRE2_SPTR substring_start = subject + ovector[2*i]; - size_t substring_length = ovector[2*i+1] - ovector[2*i]; + PCRE2_SIZE substring_length = ovector[2*i+1] - ovector[2*i]; printf("%2d: %.*s\en", i, (int)substring_length, (char *)substring_start); } diff --git a/dist2/doc/pcre2grep.1 b/dist2/doc/pcre2grep.1 index 6b3219bb..66377ce2 100644 --- a/dist2/doc/pcre2grep.1 +++ b/dist2/doc/pcre2grep.1 @@ -1,4 +1,4 @@ -.TH PCRE2GREP 1 "24 November 2018" "PCRE2 10.33" +.TH PCRE2GREP 1 "04 October 2020" "PCRE2 10.36" .SH NAME pcre2grep - a grep with Perl-compatible regular expressions. .SH SYNOPSIS @@ -79,8 +79,8 @@ matching substrings, or if \fB--only-matching\fP, \fB--file-offsets\fP, or (either shown literally, or as an offset), scanning resumes immediately following the match, so that further matches on the same line can be found. If there are multiple patterns, they are all tried on the remainder of the line, -but patterns that follow the one that matched are not tried on the earlier part -of the line. +but patterns that follow the one that matched are not tried on the earlier +matched part of the line. .P This behaviour means that the order in which multiple patterns are specified can affect the output when one of the above options is used. This is no longer @@ -115,11 +115,10 @@ ignored. .rs .sp By default, a file that contains a binary zero byte within the first 1024 bytes -is identified as a binary file, and is processed specially. (GNU grep -identifies binary files in this manner.) However, if the newline type is -specified as "nul", that is, the line terminator is a binary zero, the test for -a binary file is not applied. See the \fB--binary-files\fP option for a means -of changing the way binary files are handled. +is identified as a binary file, and is processed specially. However, if the +newline type is specified as NUL, that is, the line terminator is a binary +zero, the test for a binary file is not applied. See the \fB--binary-files\fP +option for a means of changing the way binary files are handled. . . .SH "BINARY ZEROS IN PATTERNS" @@ -383,8 +382,8 @@ Ignore upper/lower case distinctions during comparisons. .TP \fB--include\fP=\fIpattern\fP If any \fB--include\fP patterns are specified, the only files that are -processed are those that match one of the patterns (and do not match an -\fB--exclude\fP pattern). This option does not affect directories, but it +processed are those whose names match one of the patterns and do not match an +\fB--exclude\fP pattern. This option does not affect directories, but it applies to all files, whether listed on the command line, obtained from \fB--file-list\fP, or by scanning a directory. The pattern is a PCRE2 regular expression, and is matched against the final component of the file name, not @@ -401,8 +400,8 @@ may be given any number of times; all the files are read. .TP \fB--include-dir\fP=\fIpattern\fP If any \fB--include-dir\fP patterns are specified, the only directories that -are processed are those that match one of the patterns (and do not match an -\fB--exclude-dir\fP pattern). This applies to all directories, whether listed +are processed are those whose names match one of the patterns and do not match +an \fB--exclude-dir\fP pattern. This applies to all directories, whether listed on the command line, obtained from \fB--file-list\fP, or by scanning a parent directory. The pattern is a PCRE2 regular expression, and is matched against the final component of the directory name, not the entire path. The \fB-F\fP, @@ -423,8 +422,9 @@ a separate line. Searching normally stops as soon as a matching line is found in a file. However, if the \fB-c\fP (count) option is also used, matching continues in order to obtain the correct count, and those files that have at least one match are listed along with their counts. Using this option with -\fB-c\fP is a way of suppressing the listing of files with no matches. This -opeion overrides any previous \fB-H\fP, \fB-h\fP, or \fB-L\fP options. +\fB-c\fP is a way of suppressing the listing of files with no matches that +occurs with \fB-c\fP on its own. This option overrides any previous \fB-H\fP, +\fB-h\fP, or \fB-L\fP options. .TP \fB--label\fP=\fIname\fP This option supplies a name to be used for the standard input when file names @@ -435,8 +435,8 @@ short form for this option. When this option is given, non-compressed input is read and processed line by line, and the output is flushed after each write. By default, input is read in large chunks, unless \fBpcre2grep\fP can determine that it is reading from a -terminal (which is currently possible only in Unix-like environments or -Windows). Output to terminal is normally automatically flushed by the operating +terminal, which is currently possible only in Unix-like environments or +Windows. Output to terminal is normally automatically flushed by the operating system. This option can be useful when the input or output is attached to a pipe and you do not want \fBpcre2grep\fP to buffer up large amounts of data. However, its use will affect performance, and the \fB-M\fP (multiline) option @@ -459,6 +459,45 @@ the value in the \fBLC_ALL\fP or \fBLC_CTYPE\fP environment variables. If no locale is specified, the PCRE2 library's default (usually the "C" locale) is used. There is no short form for this option. .TP +\fB-M\fP, \fB--multiline\fP +Allow patterns to match more than one line. When this option is set, the PCRE2 +library is called in "multiline" mode. This allows a matched string to extend +past the end of a line and continue on one or more subsequent lines. Patterns +used with \fB-M\fP may usefully contain literal newline characters and internal +occurrences of ^ and $ characters. The output for a successful match may +consist of more than one line. The first line is the line in which the match +started, and the last line is the line in which the match ended. If the matched +string ends with a newline sequence, the output ends at the end of that line. +If \fB-v\fP is set, none of the lines in a multi-line match are output. Once a +match has been handled, scanning restarts at the beginning of the line after +the one in which the match ended. +.sp +The newline sequence that separates multiple lines must be matched as part of +the pattern. For example, to find the phrase "regular expression" in a file +where "regular" might be at the end of a line and "expression" at the start of +the next line, you could use this command: +.sp + pcre2grep -M 'regular\es+expression' <file> +.sp +The \es escape sequence matches any white space character, including newlines, +and is followed by + so as to match trailing white space on the first line as +well as possibly handling a two-character newline sequence. +.sp +There is a limit to the number of lines that can be matched, imposed by the way +that \fBpcre2grep\fP buffers the input file as it scans it. With a sufficiently +large processing buffer, this should not be a problem, but the \fB-M\fP option +does not work when input is read line by line (see \fB--line-buffered\fP.) +.TP +\fB-m\fP \fInumber\fP, \fB--max-count\fP=\fInumber\fP +Stop processing after finding \fInumber\fP matching lines, or non-matching +lines if \fB-v\fP is also set. Any trailing context lines are output after the +final match. In multiline mode, each multiline match counts as just one line +for this purpose. If this limit is reached when reading the standard input from +a regular file, the file is left positioned just after the last matching line. +If \fB-c\fP is also set, the count that is output is never greater than +\fInumber\fP. This option has no effect if used with \fB-L\fP, \fB-l\fP, or +\fB-q\fP, or when just checking for a match in a binary file. +.TP \fB--match-limit\fP=\fInumber\fP Processing some regular expression patterns may take a very long time to search for all possible matching strings. Others may require a very large amount of @@ -488,59 +527,36 @@ There are no short forms for these options. The default limits can be set when the PCRE2 library is compiled; if they are not specified, the defaults are very large and so effectively unlimited. .TP -\fB--max-buffer-size=\fInumber\fP +\fB--max-buffer-size\fP=\fInumber\fP This limits the expansion of the processing buffer, whose initial size can be set by \fB--buffer-size\fP. The maximum buffer size is silently forced to be no smaller than the starting buffer size. .TP -\fB-M\fP, \fB--multiline\fP -Allow patterns to match more than one line. When this option is set, the PCRE2 -library is called in "multiline" mode. This allows a matched string to extend -past the end of a line and continue on one or more subsequent lines. Patterns -used with \fB-M\fP may usefully contain literal newline characters and internal -occurrences of ^ and $ characters. The output for a successful match may -consist of more than one line. The first line is the line in which the match -started, and the last line is the line in which the match ended. If the matched -string ends with a newline sequence, the output ends at the end of that line. -If \fB-v\fP is set, none of the lines in a multi-line match are output. Once a -match has been handled, scanning restarts at the beginning of the line after -the one in which the match ended. +\fB-N\fP \fInewline-type\fP, \fB--newline\fP=\fInewline-type\fP +Six different conventions for indicating the ends of lines in scanned files are +supported. For example: .sp -The newline sequence that separates multiple lines must be matched as part of -the pattern. For example, to find the phrase "regular expression" in a file -where "regular" might be at the end of a line and "expression" at the start of -the next line, you could use this command: + pcre2grep -N CRLF 'some pattern' <file> .sp - pcre2grep -M 'regular\es+expression' <file> -.sp -The \es escape sequence matches any white space character, including newlines, -and is followed by + so as to match trailing white space on the first line as -well as possibly handling a two-character newline sequence. -.sp -There is a limit to the number of lines that can be matched, imposed by the way -that \fBpcre2grep\fP buffers the input file as it scans it. With a sufficiently -large processing buffer, this should not be a problem, but the \fB-M\fP option -does not work when input is read line by line (see \fP--line-buffered\fP.) -.TP -\fB-N\fP \fInewline-type\fP, \fB--newline\fP=\fInewline-type\fP -The PCRE2 library supports five different conventions for indicating -the ends of lines. They are the single-character sequences CR (carriage return) -and LF (linefeed), the two-character sequence CRLF, an "anycrlf" convention, -which recognizes any of the preceding three types, and an "any" convention, in -which any Unicode line ending sequence is assumed to end a line. The Unicode -sequences are the three just mentioned, plus VT (vertical tab, U+000B), FF -(form feed, U+000C), NEL (next line, U+0085), LS (line separator, U+2028), and -PS (paragraph separator, U+2029). +The newline type may be specified in upper, lower, or mixed case. If the +newline type is NUL, lines are separated by binary zero characters. The other +types are the single-character sequences CR (carriage return) and LF +(linefeed), the two-character sequence CRLF, an "anycrlf" type, which +recognizes any of the preceding three types, and an "any" type, for which any +Unicode line ending sequence is assumed to end a line. The Unicode sequences +are the three just mentioned, plus VT (vertical tab, U+000B), FF (form feed, +U+000C), NEL (next line, U+0085), LS (line separator, U+2028), and PS +(paragraph separator, U+2029). .sp When the PCRE2 library is built, a default line-ending sequence is specified. This is normally the standard sequence for the operating system. Unless otherwise specified by this option, \fBpcre2grep\fP uses the library's default. -The possible values for this option are CR, LF, CRLF, ANYCRLF, or ANY. This -makes it possible to use \fBpcre2grep\fP to scan files that have come from -other environments without having to modify their line endings. If the data -that is being scanned does not agree with the convention set by this option, -\fBpcre2grep\fP may behave in strange ways. Note that this option does not -apply to files specified by the \fB-f\fP, \fB--exclude-from\fP, or +.sp +This option makes it possible to use \fBpcre2grep\fP to scan files that have +come from other environments without having to modify their line endings. If +the data that is being scanned does not agree with the convention set by this +option, \fBpcre2grep\fP may behave in strange ways. Note that this option does +not apply to files specified by the \fB-f\fP, \fB--exclude-from\fP, or \fB--include-from\fP options, which are expected to use the operating system's standard newline sequence. .TP @@ -559,25 +575,36 @@ use of JIT at run time. It is provided for testing and working round problems. It should never be needed in normal use. .TP \fB-O\fP \fItext\fP, \fB--output\fP=\fItext\fP -When there is a match, instead of outputting the whole line that matched, -output just the given text. This option is mutually exclusive with -\fB--only-matching\fP, \fB--file-offsets\fP, and \fB--line-offsets\fP. Escape -sequences starting with a dollar character may be used to insert the contents -of the matched part of the line and/or captured substrings into the text. -.sp -$<digits> or ${<digits>} is replaced by the captured -substring of the given decimal number; zero substitutes the whole match. If -the number is greater than the number of capturing substrings, or if the -capture is unset, the replacement is empty. +When there is a match, instead of outputting the line that matched, output just +the text specified in this option, followed by an operating-system standard +newline. In this mode, no context is shown. That is, the \fB-A\fP, \fB-B\fP, +and \fB-C\fP options are ignored. The \fB--newline\fP option has no effect on +this option, which is mutually exclusive with \fB--only-matching\fP, +\fB--file-offsets\fP, and \fB--line-offsets\fP. However, like +\fB--only-matching\fP, if there is more than one match in a line, each of them +causes a line of output. +.sp +Escape sequences starting with a dollar character may be used to insert the +contents of the matched part of the line and/or captured substrings into the +text. +.sp +$<digits> or ${<digits>} is replaced by the captured substring of the given +decimal number; zero substitutes the whole match. If the number is greater than +the number of capturing substrings, or if the capture is unset, the replacement +is empty. .sp $a is replaced by bell; $b by backspace; $e by escape; $f by form feed; $n by newline; $r by carriage return; $t by tab; $v by vertical tab. .sp -$o<digits> is replaced by the character represented by the given octal -number; up to three digits are processed. +$o<digits> or $o{<digits>} is replaced by the character whose code point is the +given octal number. In the first form, up to three octal digits are processed. +When more digits are needed in Unicode mode to specify a wide character, the +second form must be used. .sp -$x<digits> is replaced by the character represented by the given hexadecimal -number; up to two digits are processed. +$x<digits> or $x{<digits>} is replaced by the character represented by the +given hexadecimal number. In the first form, up to two hexadecimal digits are +processed. When more digits are needed in Unicode mode to specify a wide +character, the second form must be used. .sp Any other character is substituted by itself. In particular, $$ is replaced by a single dollar. @@ -596,19 +623,29 @@ otherwise empty line. This option is mutually exclusive with \fB--output\fP, .TP \fB-o\fP\fInumber\fP, \fB--only-matching\fP=\fInumber\fP Show only the part of the line that matched the capturing parentheses of the -given number. Up to 32 capturing parentheses are supported, and -o0 is -equivalent to \fB-o\fP without a number. Because these options can be given -without an argument (see above), if an argument is present, it must be given in -the same shell item, for example, -o3 or --only-matching=2. The comments given -for the non-argument case above also apply to this option. If the specified -capturing parentheses do not exist in the pattern, or were not set in the -match, nothing is output unless the file name or line number are being output. +given number. Up to 50 capturing parentheses are supported by default. This +limit can be changed via the \fB--om-capture\fP option. A pattern may contain +any number of capturing parentheses, but only those whose number is within the +limit can be accessed by \fB-o\fP. An error occurs if the number specified by +\fB-o\fP is greater than the limit. +.sp +-o0 is the same as \fB-o\fP without a number. Because these options can be +given without an argument (see above), if an argument is present, it must be +given in the same shell item, for example, -o3 or --only-matching=2. The +comments given for the non-argument case above also apply to this option. If +the specified capturing parentheses do not exist in the pattern, or were not +set in the match, nothing is output unless the file name or line number are +being output. .sp If this option is given multiple times, multiple substrings are output for each match, in the order the options are given, and all on one line. For example, -o3 -o1 -o3 causes the substrings matched by capturing parentheses 3 and 1 and then 3 again to be output. By default, there is no separator (but see the next -option). +but one option). +.TP +\fB--om-capture\fP=\fInumber\fP +Set the number of capturing parentheses that can be accessed by \fB-o\fP. The +default is 50. .TP \fB--om-separator\fP=\fItext\fP Specify a separating string for multiple occurrences of \fB-o\fP. The default @@ -626,7 +663,8 @@ immediate end-of-file. This option is a shorthand for setting the \fB-d\fP option to "recurse". .TP \fB--recursion-limit\fP=\fInumber\fP -See \fB--match-limit\fP above. +This is an obsolete synonym for \fB--depth-limit\fP. See \fB--match-limit\fP +above for details. .TP \fB-s\fP, \fB--no-messages\fP Suppress error messages about non-existent or unreadable files. Such files are @@ -644,11 +682,24 @@ is listed. If file names are being output, the grand total is preceded by ignored when used with \fB-L\fP (list files without matches), because the grand total would always be zero. .TP -\fB-u\fP, \fB--utf-8\fP +\fB-u\fP, \fB--utf\fP Operate in UTF-8 mode. This option is available only if PCRE2 has been compiled with UTF-8 support. All patterns (including those for any \fB--exclude\fP and -\fB--include\fP options) and all subject lines that are scanned must be valid -strings of UTF-8 characters. +\fB--include\fP options) and all lines that are scanned must be valid strings +of UTF-8 characters. If an invalid UTF-8 string is encountered, an error +occurs. +.TP +\fB-U\fP, \fB--utf-allow-invalid\fP +As \fB--utf\fP, but in addition subject lines may contain invalid UTF-8 code +unit sequences. These can never form part of any pattern match. Patterns +themselves, however, must still be valid UTF-8 strings. This facility allows +valid UTF-8 strings to be sought within arbitrary byte sequences in executable +or other binary files. For more details about matching in non-valid UTF-8 +strings, see the +.\" HREF +\fBpcre2unicode\fP(3) +.\" +documentation. .TP \fB-V\fP, \fB--version\fP Write the version numbers of \fBpcre2grep\fP and the PCRE2 library to the @@ -657,7 +708,9 @@ ignored. .TP \fB-v\fP, \fB--invert-match\fP Invert the sense of the match, so that lines which do \fInot\fP match any of -the patterns are the ones that are found. +the patterns are the ones that are found. When this option is set, options such +as \fB--only-matching\fP and \fB--output\fP, which specify parts of a match +that are to be output, are ignored. .TP \fB-w\fP, \fB--word-regex\fP, \fB--word-regexp\fP Force the patterns only to match "words". That is, there must be a word @@ -689,16 +742,25 @@ by the \fB--locale\fP option. If no locale is set, the PCRE2 library's default .rs .sp The \fB-N\fP (\fB--newline\fP) option allows \fBpcre2grep\fP to scan files with -different newline conventions from the default. Any parts of the input files -that are written to the standard output are copied identically, with whatever -newline sequences they have in the input. However, the setting of this option -affects only the way scanned files are processed. It does not affect the -interpretation of files specified by the \fB-f\fP, \fB--file-list\fP, -\fB--exclude-from\fP, or \fB--include-from\fP options, nor does it affect the -way in which \fBpcre2grep\fP writes informational messages to the standard -error and output streams. For these it uses the string "\en" to indicate -newlines, relying on the C I/O library to convert this to an appropriate -sequence. +newline conventions that differ from the default. This option affects only the +way scanned files are processed. It does not affect the interpretation of files +specified by the \fB-f\fP, \fB--file-list\fP, \fB--exclude-from\fP, or +\fB--include-from\fP options. +.P +Any parts of the scanned input files that are written to the standard output +are copied with whatever newline sequences they have in the input. However, if +the final line of a file is output, and it does not end with a newline +sequence, a newline sequence is added. If the newline setting is CR, LF, CRLF +or NUL, that line ending is output; for the other settings (ANYCRLF or ANY) a +single NL is used. +.P +The newline setting does not affect the way in which \fBpcre2grep\fP writes +newlines in informational messages to the standard output and error streams. +Under Windows, the standard output is set to be binary, so that "\er\en" at the +ends of output lines that are copied from the input is not converted to +"\er\er\en" by the C I/O library. This means that any messages written to the +standard output must end with "\er\en". For all other operating systems, and +for all messages to the standard error stream, "\en" is used. . . .SH "OPTIONS COMPATIBILITY" @@ -711,9 +773,9 @@ as in the GNU \fBgrep\fP program. Any long option of the form \fB--file-offsets\fP, \fB--heap-limit\fP, \fB--include-dir\fP, \fB--line-offsets\fP, \fB--locale\fP, \fB--match-limit\fP, \fB-M\fP, \fB--multiline\fP, \fB-N\fP, \fB--newline\fP, \fB--om-separator\fP, -\fB--output\fP, \fB-u\fP, and \fB--utf-8\fP options are specific to -\fBpcre2grep\fP, as is the use of the \fB--only-matching\fP option with a -capturing parentheses number. +\fB--output\fP, \fB-u\fP, \fB--utf\fP, \fB-U\fP, and \fB--utf-allow-invalid\fP +options are specific to \fBpcre2grep\fP, as is the use of the +\fB--only-matching\fP option with a capturing parentheses number. .P Although most of the common options work the same way, a few are different in \fBpcre2grep\fP. For example, the \fB--include\fP option's argument is a glob @@ -775,12 +837,36 @@ documentation for details). Numbered callouts are ignored by \fBpcre2grep\fP; only callouts with string arguments are useful. . . +.SS "Echoing a specific string" +.rs +.sp +Starting the callout string with a pipe character invokes an echoing facility +that avoids calling an external program or script. This facility is always +available, provided that callouts were not completely disabled when +\fBpcre2grep\fP was built. The rest of the callout string is processed as a +zero-terminated string, which means it should not contain any internal binary +zeros. It is written to the output, having first been passed through the same +escape processing as text from the \fB--output\fP (\fB-O\fP) option (see +above). However, $0 cannot be used to insert a matched substring because the +match is still in progress. Instead, the single character '0' is inserted. Any +syntax errors in the string (for example, a dollar not followed by another +character) causes the callout to be ignored. No terminator is added to the +output string, so if you want a newline, you must include it explicitly using +the escape $n. For example: +.sp + pcre2grep '(.)(..(.))(?C"|[$1] [$2] [$3]$n")' <some file> +.sp +Matching continues normally after the string is output. If you want to see only +the callout output but not any output from an actual match, you should end the +pattern with (*FAIL). +. +. .SS "Calling external programs or scripts" .rs .sp This facility can be independently disabled when \fBpcre2grep\fP is built. It is supported for Windows, where a call to \fB_spawnvp()\fP is used, for VMS, -where \fBlib$spawn()\fP is used, and for any other Unix-like environment where +where \fBlib$spawn()\fP is used, and for any Unix-like environment where \fBfork()\fP and \fBexecv()\fP are available. .P If the callout string does not start with a pipe (vertical bar) character, it @@ -791,13 +877,11 @@ arguments: executable_name|arg1|arg2|... .sp Any substring (including the executable name) may contain escape sequences -started by a dollar character: $<digits> or ${<digits>} is replaced by the -captured substring of the given decimal number, which must be greater than -zero. If the number is greater than the number of capturing substrings, or if -the capture is unset, the replacement is empty. -.P -Any other character is substituted by itself. In particular, $$ is replaced by -a single dollar and $| is replaced by a pipe character. Here is an example: +started by a dollar character. These are the same as for the \fB--output\fP +(\fB-O\fP) option documented above, except that $0 cannot insert the matched +string because the match is still in progress. Instead, the character '0' +is inserted. If you need a literal dollar or pipe character in any +substring, use $$ or $| respectively. Here is an example: .sp echo -e "abcde\en12345" | pcre2grep \e '(?x)(.)(..(.)) @@ -810,28 +894,14 @@ a single dollar and $| is replaced by a pipe character. Here is an example: Arg1: [1] [234] [4] Arg2: |1| () 12345 .sp -The parameters for the system call that is used to run the -program or script are zero-terminated strings. This means that binary zero -characters in the callout argument will cause premature termination of their -substrings, and therefore should not be present. Any syntax errors in the -string (for example, a dollar not followed by another character) cause the -callout to be ignored. If running the program fails for any reason (including -the non-existence of the executable), a local matching failure occurs and the -matcher backtracks in the normal way. -. -. -.SS "Echoing a specific string" -.rs -.sp -This facility is always available, provided that callouts were not completely -disabled when \fBpcre2grep\fP was built. If the callout string starts with a -pipe (vertical bar) character, the rest of the string is written to the output, -having been passed through the same escape processing as text from the --output -option. This provides a simple echoing facility that avoids calling an external -program or script. No terminator is added to the string, so if you want a -newline, you must include it explicitly. Matching continues normally after the -string is output. If you want to see only the callout output but not any output -from an actual match, you should end the relevant pattern with (*FAIL). +The parameters for the system call that is used to run the program or script +are zero-terminated strings. This means that binary zero characters in the +callout argument will cause premature termination of their substrings, and +therefore should not be present. Any syntax errors in the string (for example, +a dollar not followed by another character) causes the callout to be ignored. +If running the program fails for any reason (including the non-existence of the +executable), a local matching failure occurs and the matcher backtracks in the +normal way. . . .SH "MATCHING ERRORS" @@ -867,7 +937,8 @@ because VMS does not distinguish between exit(0) and exit(1). .SH "SEE ALSO" .rs .sp -\fBpcre2pattern\fP(3), \fBpcre2syntax\fP(3), \fBpcre2callout\fP(3). +\fBpcre2pattern\fP(3), \fBpcre2syntax\fP(3), \fBpcre2callout\fP(3), +\fBpcre2unicode\fP(3). . . .SH AUTHOR @@ -884,6 +955,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 24 November 2018 -Copyright (c) 1997-2018 University of Cambridge. +Last updated: 04 October 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/doc/pcre2grep.txt b/dist2/doc/pcre2grep.txt index cd44fe00..0e839c70 100644 --- a/dist2/doc/pcre2grep.txt +++ b/dist2/doc/pcre2grep.txt @@ -12,11 +12,11 @@ SYNOPSIS DESCRIPTION pcre2grep searches files for character patterns, in the same way as - other grep commands do, but it uses the PCRE2 regular expression - library to support patterns that are compatible with the regular - expressions of Perl 5. See pcre2syntax(3) for a quick-reference summary - of pattern syntax, or pcre2pattern(3) for a full description of the - syntax and semantics of the regular expressions that PCRE2 supports. + other grep commands do, but it uses the PCRE2 regular expression li- + brary to support patterns that are compatible with the regular expres- + sions of Perl 5. See pcre2syntax(3) for a quick-reference summary of + pattern syntax, or pcre2pattern(3) for a full description of the syntax + and semantics of the regular expressions that PCRE2 supports. Patterns, whether supplied on the command line or in a separate file, are given without delimiters. For example: @@ -26,8 +26,8 @@ DESCRIPTION If you attempt to use delimiters (for example, by surrounding a pattern with slashes, as is common in Perl scripts), they are interpreted as part of the pattern. Quotes can of course be used to delimit patterns - on the command line because they are interpreted by the shell, and - indeed quotes are required if a pattern contains white space or shell + on the command line because they are interpreted by the shell, and in- + deed quotes are required if a pattern contains white space or shell metacharacters. The first argument that follows any option settings is treated as the @@ -54,8 +54,8 @@ DESCRIPTION controlled by parameters that can be set by the --buffer-size and --max-buffer-size options. The first of these sets the size of buffer that is obtained at the start of processing. If an input file contains - very long lines, a larger buffer may be needed; this is handled by - automatically extending the buffer, up to the limit specified by --max- + very long lines, a larger buffer may be needed; this is handled by au- + tomatically extending the buffer, up to the limit specified by --max- buffer-size. The default values for these parameters can be set when pcre2grep is built; if nothing is specified, the defaults are set to 20KiB and 1MiB respectively. An error occurs if a line is too long and @@ -75,12 +75,12 @@ DESCRIPTION By default, as soon as one pattern matches a line, no further patterns are considered. However, if --colour (or --color) is used to colour the matching substrings, or if --only-matching, --file-offsets, or --line- - offsets is used to output only the part of the line that matched - (either shown literally, or as an offset), scanning resumes immediately + offsets is used to output only the part of the line that matched (ei- + ther shown literally, or as an offset), scanning resumes immediately following the match, so that further matches on the same line can be - found. If there are multiple patterns, they are all tried on the - remainder of the line, but patterns that follow the one that matched - are not tried on the earlier part of the line. + found. If there are multiple patterns, they are all tried on the re- + mainder of the line, but patterns that follow the one that matched are + not tried on the earlier matched part of the line. This behaviour means that the order in which multiple patterns are specified can affect the output when one of the above options is used. @@ -89,11 +89,11 @@ DESCRIPTION overlap). Patterns that can match an empty string are accepted, but empty string - matches are never recognized. An example is the pattern - "(super)?(man)?", in which all components are optional. This pattern - finds all occurrences of both "super" and "man"; the output differs - from matching with "super|man" when only the matching substrings are - being shown. + matches are never recognized. An example is the pattern "(su- + per)?(man)?", in which all components are optional. This pattern finds + all occurrences of both "super" and "man"; the output differs from + matching with "super|man" when only the matching substrings are being + shown. If the LC_ALL or LC_CTYPE environment variable is set, pcre2grep uses the value to set a locale when calling the PCRE2 library. The --locale @@ -115,154 +115,153 @@ BINARY FILES By default, a file that contains a binary zero byte within the first 1024 bytes is identified as a binary file, and is processed specially. - (GNU grep identifies binary files in this manner.) However, if the new- - line type is specified as "nul", that is, the line terminator is a - binary zero, the test for a binary file is not applied. See the - --binary-files option for a means of changing the way binary files are - handled. + However, if the newline type is specified as NUL, that is, the line + terminator is a binary zero, the test for a binary file is not applied. + See the --binary-files option for a means of changing the way binary + files are handled. BINARY ZEROS IN PATTERNS - Patterns passed from the command line are strings that are terminated - by a binary zero, so cannot contain internal zeros. However, patterns + Patterns passed from the command line are strings that are terminated + by a binary zero, so cannot contain internal zeros. However, patterns that are read from a file via the -f option may contain binary zeros. OPTIONS - The order in which some of the options appear can affect the output. - For example, both the -H and -l options affect the printing of file - names. Whichever comes later in the command line will be the one that - takes effect. Similarly, except where noted below, if an option is - given twice, the later setting is used. Numerical values for options - may be followed by K or M, to signify multiplication by 1024 or + The order in which some of the options appear can affect the output. + For example, both the -H and -l options affect the printing of file + names. Whichever comes later in the command line will be the one that + takes effect. Similarly, except where noted below, if an option is + given twice, the later setting is used. Numerical values for options + may be followed by K or M, to signify multiplication by 1024 or 1024*1024 respectively. -- This terminates the list of options. It is useful if the next - item on the command line starts with a hyphen but is not an - option. This allows for the processing of patterns and file + item on the command line starts with a hyphen but is not an + option. This allows for the processing of patterns and file names that start with hyphens. -A number, --after-context=number - Output up to number lines of context after each matching - line. Fewer lines are output if the next match or the end of - the file is reached, or if the processing buffer size has - been set too small. If file names and/or line numbers are - being output, a hyphen separator is used instead of a colon - for the context lines. A line containing "--" is output - between each group of lines, unless they are in fact contigu- - ous in the input file. The value of number is expected to be - relatively small. When -c is used, -A is ignored. + Output up to number lines of context after each matching + line. Fewer lines are output if the next match or the end of + the file is reached, or if the processing buffer size has + been set too small. If file names and/or line numbers are be- + ing output, a hyphen separator is used instead of a colon for + the context lines. A line containing "--" is output between + each group of lines, unless they are in fact contiguous in + the input file. The value of number is expected to be rela- + tively small. When -c is used, -A is ignored. -a, --text - Treat binary files as text. This is equivalent to --binary- + Treat binary files as text. This is equivalent to --binary- files=text. -B number, --before-context=number - Output up to number lines of context before each matching - line. Fewer lines are output if the previous match or the - start of the file is within number lines, or if the process- - ing buffer size has been set too small. If file names and/or - line numbers are being output, a hyphen separator is used - instead of a colon for the context lines. A line containing - "--" is output between each group of lines, unless they are - in fact contiguous in the input file. The value of number is - expected to be relatively small. When -c is used, -B is - ignored. + Output up to number lines of context before each matching + line. Fewer lines are output if the previous match or the + start of the file is within number lines, or if the process- + ing buffer size has been set too small. If file names and/or + line numbers are being output, a hyphen separator is used in- + stead of a colon for the context lines. A line containing + "--" is output between each group of lines, unless they are + in fact contiguous in the input file. The value of number is + expected to be relatively small. When -c is used, -B is ig- + nored. --binary-files=word - Specify how binary files are to be processed. If the word is - "binary" (the default), pattern matching is performed on - binary files, but the only output is "Binary file <name> - matches" when a match succeeds. If the word is "text", which - is equivalent to the -a or --text option, binary files are - processed in the same way as any other file. In this case, - when a match succeeds, the output may be binary garbage, - which can have nasty effects if sent to a terminal. If the - word is "without-match", which is equivalent to the -I - option, binary files are not processed at all; they are - assumed not to be of interest and are skipped without causing - any output or affecting the return code. + Specify how binary files are to be processed. If the word is + "binary" (the default), pattern matching is performed on bi- + nary files, but the only output is "Binary file <name> + matches" when a match succeeds. If the word is "text", which + is equivalent to the -a or --text option, binary files are + processed in the same way as any other file. In this case, + when a match succeeds, the output may be binary garbage, + which can have nasty effects if sent to a terminal. If the + word is "without-match", which is equivalent to the -I op- + tion, binary files are not processed at all; they are assumed + not to be of interest and are skipped without causing any + output or affecting the return code. --buffer-size=number - Set the parameter that controls how much memory is obtained + Set the parameter that controls how much memory is obtained at the start of processing for buffering files that are being scanned. See also --max-buffer-size below. -C number, --context=number - Output number lines of context both before and after each - matching line. This is equivalent to setting both -A and -B + Output number lines of context both before and after each + matching line. This is equivalent to setting both -A and -B to the same value. -c, --count - Do not output lines from the files that are being scanned; - instead output the number of lines that would have been + Do not output lines from the files that are being scanned; + instead output the number of lines that would have been shown, either because they matched, or, if -v is set, because - they failed to match. By default, this count is exactly the - same as the number of lines that would have been output, but - if the -M (multiline) option is used (without -v), there may - be more suppressed lines than the count (that is, the number + they failed to match. By default, this count is exactly the + same as the number of lines that would have been output, but + if the -M (multiline) option is used (without -v), there may + be more suppressed lines than the count (that is, the number of matches). - If no lines are selected, the number zero is output. If sev- - eral files are are being scanned, a count is output for each - of them and the -t option can be used to cause a total to be - output at the end. However, if the --files-with-matches - option is also used, only those files whose counts are - greater than zero are listed. When -c is used, the -A, -B, - and -C options are ignored. + If no lines are selected, the number zero is output. If sev- + eral files are are being scanned, a count is output for each + of them and the -t option can be used to cause a total to be + output at the end. However, if the --files-with-matches op- + tion is also used, only those files whose counts are greater + than zero are listed. When -c is used, the -A, -B, and -C op- + tions are ignored. --colour, --color If this option is given without any data, it is equivalent to - "--colour=auto". If data is required, it must be given in + "--colour=auto". If data is required, it must be given in the same shell item, separated by an equals sign. --colour=value, --color=value This option specifies under what circumstances the parts of a line that matched a pattern should be coloured in the output. - By default, the output is not coloured. The value (which is - optional, see above) may be "never", "always", or "auto". In - the latter case, colouring happens only if the standard out- - put is connected to a terminal. More resources are used when + By default, the output is not coloured. The value (which is + optional, see above) may be "never", "always", or "auto". In + the latter case, colouring happens only if the standard out- + put is connected to a terminal. More resources are used when colouring is enabled, because pcre2grep has to search for all - possible matches in a line, not just one, in order to colour + possible matches in a line, not just one, in order to colour them all. - The colour that is used can be specified by setting one of - the environment variables PCRE2GREP_COLOUR, PCRE2GREP_COLOR, + The colour that is used can be specified by setting one of + the environment variables PCRE2GREP_COLOUR, PCRE2GREP_COLOR, PCREGREP_COLOUR, or PCREGREP_COLOR, which are checked in that order. If none of these are set, pcre2grep looks for - GREP_COLORS or GREP_COLOR (in that order). The value of the - variable should be a string of two numbers, separated by a - semicolon, except in the case of GREP_COLORS, which must + GREP_COLORS or GREP_COLOR (in that order). The value of the + variable should be a string of two numbers, separated by a + semicolon, except in the case of GREP_COLORS, which must start with "ms=" or "mt=" followed by two semicolon-separated - colours, terminated by the end of the string or by a colon. - If GREP_COLORS does not start with "ms=" or "mt=" it is - ignored, and GREP_COLOR is checked. + colours, terminated by the end of the string or by a colon. + If GREP_COLORS does not start with "ms=" or "mt=" it is ig- + nored, and GREP_COLOR is checked. - If the string obtained from one of the above variables con- + If the string obtained from one of the above variables con- tains any characters other than semicolon or digits, the set- ting is ignored and the default colour is used. The string is copied directly into the control string for setting colour on - a terminal, so it is your responsibility to ensure that the - values make sense. If no relevant environment variable is + a terminal, so it is your responsibility to ensure that the + values make sense. If no relevant environment variable is set, the default is "1;31", which gives red. -D action, --devices=action - If an input path is not a regular file or a directory, - "action" specifies how it is to be processed. Valid values - are "read" (the default) or "skip" (silently skip the path). + If an input path is not a regular file or a directory, "ac- + tion" specifies how it is to be processed. Valid values are + "read" (the default) or "skip" (silently skip the path). -d action, --directories=action If an input path is a directory, "action" specifies how it is - to be processed. Valid values are "read" (the default in - non-Windows environments, for compatibility with GNU grep), - "recurse" (equivalent to the -r option), or "skip" (silently - skip the path, the default in Windows environments). In the - "read" case, directories are read as if they were ordinary - files. In some operating systems the effect of reading a - directory like this is an immediate end-of-file; in others it + to be processed. Valid values are "read" (the default in + non-Windows environments, for compatibility with GNU grep), + "recurse" (equivalent to the -r option), or "skip" (silently + skip the path, the default in Windows environments). In the + "read" case, directories are read as if they were ordinary + files. In some operating systems the effect of reading a di- + rectory like this is an immediate end-of-file; in others it may provoke an error. --depth-limit=number @@ -271,199 +270,200 @@ OPTIONS -e pattern, --regex=pattern, --regexp=pattern Specify a pattern to be matched. This option can be used mul- tiple times in order to specify several patterns. It can also - be used as a way of specifying a single pattern that starts - with a hyphen. When -e is used, no argument pattern is taken - from the command line; all arguments are treated as file - names. There is no limit to the number of patterns. They are - applied to each line in the order in which they are defined + be used as a way of specifying a single pattern that starts + with a hyphen. When -e is used, no argument pattern is taken + from the command line; all arguments are treated as file + names. There is no limit to the number of patterns. They are + applied to each line in the order in which they are defined until one matches. - If -f is used with -e, the command line patterns are matched + If -f is used with -e, the command line patterns are matched first, followed by the patterns from the file(s), independent - of the order in which these options are specified. Note that - multiple use of -e is not the same as a single pattern with + of the order in which these options are specified. Note that + multiple use of -e is not the same as a single pattern with alternatives. For example, X|Y finds the first character in a - line that is X or Y, whereas if the two patterns are given + line that is X or Y, whereas if the two patterns are given separately, with X first, pcre2grep finds X if it is present, even if it follows Y in the line. It finds Y only if there is - no X in the line. This matters only if you are using -o or + no X in the line. This matters only if you are using -o or --colo(u)r to show the part(s) of the line that matched. --exclude=pattern Files (but not directories) whose names match the pattern are - skipped without being processed. This applies to all files, - whether listed on the command line, obtained from --file- + skipped without being processed. This applies to all files, + whether listed on the command line, obtained from --file- list, or by scanning a directory. The pattern is a PCRE2 reg- - ular expression, and is matched against the final component - of the file name, not the entire path. The -F, -w, and -x - options do not apply to this pattern. The option may be given + ular expression, and is matched against the final component + of the file name, not the entire path. The -F, -w, and -x op- + tions do not apply to this pattern. The option may be given any number of times in order to specify multiple patterns. If - a file name matches both an --include and an --exclude pat- + a file name matches both an --include and an --exclude pat- tern, it is excluded. There is no short form for this option. --exclude-from=filename - Treat each non-empty line of the file as the data for an + Treat each non-empty line of the file as the data for an --exclude option. What constitutes a newline when reading the - file is the operating system's default. The --newline option - has no effect on this option. This option may be given more + file is the operating system's default. The --newline option + has no effect on this option. This option may be given more than once in order to specify a number of files to read. --exclude-dir=pattern Directories whose names match the pattern are skipped without - being processed, whatever the setting of the --recursive - option. This applies to all directories, whether listed on - the command line, obtained from --file-list, or by scanning a - parent directory. The pattern is a PCRE2 regular expression, - and is matched against the final component of the directory - name, not the entire path. The -F, -w, and -x options do not - apply to this pattern. The option may be given any number of - times in order to specify more than one pattern. If a direc- - tory matches both --include-dir and --exclude-dir, it is - excluded. There is no short form for this option. + being processed, whatever the setting of the --recursive op- + tion. This applies to all directories, whether listed on the + command line, obtained from --file-list, or by scanning a + parent directory. The pattern is a PCRE2 regular expression, + and is matched against the final component of the directory + name, not the entire path. The -F, -w, and -x options do not + apply to this pattern. The option may be given any number of + times in order to specify more than one pattern. If a direc- + tory matches both --include-dir and --exclude-dir, it is ex- + cluded. There is no short form for this option. -F, --fixed-strings - Interpret each data-matching pattern as a list of fixed - strings, separated by newlines, instead of as a regular - expression. What constitutes a newline for this purpose is - controlled by the --newline option. The -w (match as a word) - and -x (match whole line) options can be used with -F. They - apply to each of the fixed strings. A line is selected if any + Interpret each data-matching pattern as a list of fixed + strings, separated by newlines, instead of as a regular ex- + pression. What constitutes a newline for this purpose is con- + trolled by the --newline option. The -w (match as a word) and + -x (match whole line) options can be used with -F. They ap- + ply to each of the fixed strings. A line is selected if any of the fixed strings are found in it (subject to -w or -x, if - present). This option applies only to the patterns that are - matched against the contents of files; it does not apply to - patterns specified by any of the --include or --exclude - options. + present). This option applies only to the patterns that are + matched against the contents of files; it does not apply to + patterns specified by any of the --include or --exclude op- + tions. -f filename, --file=filename - Read patterns from the file, one per line, and match them - against each line of input. As is the case with patterns on - the command line, no delimiters should be used. What consti- - tutes a newline when reading the file is the operating sys- - tem's default interpretation of \n. The --newline option has - no effect on this option. Trailing white space is removed - from each line, and blank lines are ignored. An empty file - contains no patterns and therefore matches nothing. Patterns - read from a file in this way may contain binary zeros, which - are treated as ordinary data characters. See also the com- - ments about multiple patterns versus a single pattern with + Read patterns from the file, one per line, and match them + against each line of input. As is the case with patterns on + the command line, no delimiters should be used. What consti- + tutes a newline when reading the file is the operating sys- + tem's default interpretation of \n. The --newline option has + no effect on this option. Trailing white space is removed + from each line, and blank lines are ignored. An empty file + contains no patterns and therefore matches nothing. Patterns + read from a file in this way may contain binary zeros, which + are treated as ordinary data characters. See also the com- + ments about multiple patterns versus a single pattern with alternatives in the description of -e above. - If this option is given more than once, all the specified - files are read. A data line is output if any of the patterns - match it. A file name can be given as "-" to refer to the - standard input. When -f is used, patterns specified on the - command line using -e may also be present; they are tested - before the file's patterns. However, no other pattern is + If this option is given more than once, all the specified + files are read. A data line is output if any of the patterns + match it. A file name can be given as "-" to refer to the + standard input. When -f is used, patterns specified on the + command line using -e may also be present; they are tested + before the file's patterns. However, no other pattern is taken from the command line; all arguments are treated as the names of paths to be searched. --file-list=filename - Read a list of files and/or directories that are to be + Read a list of files and/or directories that are to be scanned from the given file, one per line. What constitutes a - newline when reading the file is the operating system's - default. Trailing white space is removed from each line, and + newline when reading the file is the operating system's de- + fault. Trailing white space is removed from each line, and blank lines are ignored. These paths are processed before any - that are listed on the command line. The file name can be - given as "-" to refer to the standard input. If --file and - --file-list are both specified as "-", patterns are read - first. This is useful only when the standard input is a ter- - minal, from which further lines (the list of files) can be + that are listed on the command line. The file name can be + given as "-" to refer to the standard input. If --file and + --file-list are both specified as "-", patterns are read + first. This is useful only when the standard input is a ter- + minal, from which further lines (the list of files) can be read after an end-of-file indication. If this option is given more than once, all the specified files are read. --file-offsets - Instead of showing lines or parts of lines that match, show - each match as an offset from the start of the file and a - length, separated by a comma. In this mode, no context is - shown. That is, the -A, -B, and -C options are ignored. If + Instead of showing lines or parts of lines that match, show + each match as an offset from the start of the file and a + length, separated by a comma. In this mode, no context is + shown. That is, the -A, -B, and -C options are ignored. If there is more than one match in a line, each of them is shown - separately. This option is mutually exclusive with --output, + separately. This option is mutually exclusive with --output, --line-offsets, and --only-matching. -H, --with-filename - Force the inclusion of the file name at the start of output + Force the inclusion of the file name at the start of output lines when searching a single file. By default, the file name is not shown in this case. For matching lines, the file name is followed by a colon; for context lines, a hyphen separator - is used. If a line number is also being output, it follows - the file name. When the -M option causes a pattern to match - more than one line, only the first is preceded by the file - name. This option overrides any previous -h, -l, or -L - options. + is used. If a line number is also being output, it follows + the file name. When the -M option causes a pattern to match + more than one line, only the first is preceded by the file + name. This option overrides any previous -h, -l, or -L op- + tions. -h, --no-filename Suppress the output file names when searching multiple files. - By default, file names are shown when multiple files are - searched. For matching lines, the file name is followed by a - colon; for context lines, a hyphen separator is used. If a - line number is also being output, it follows the file name. + By default, file names are shown when multiple files are + searched. For matching lines, the file name is followed by a + colon; for context lines, a hyphen separator is used. If a + line number is also being output, it follows the file name. This option overrides any previous -H, -L, or -l options. --heap-limit=number See --match-limit below. - --help Output a help message, giving brief details of the command - options and file type support, and then exit. Anything else + --help Output a help message, giving brief details of the command + options and file type support, and then exit. Anything else on the command line is ignored. - -I Ignore binary files. This is equivalent to --binary- + -I Ignore binary files. This is equivalent to --binary- files=without-match. -i, --ignore-case Ignore upper/lower case distinctions during comparisons. --include=pattern - If any --include patterns are specified, the only files that - are processed are those that match one of the patterns (and - do not match an --exclude pattern). This option does not - affect directories, but it applies to all files, whether - listed on the command line, obtained from --file-list, or by - scanning a directory. The pattern is a PCRE2 regular expres- - sion, and is matched against the final component of the file - name, not the entire path. The -F, -w, and -x options do not - apply to this pattern. The option may be given any number of - times. If a file name matches both an --include and an - --exclude pattern, it is excluded. There is no short form - for this option. + If any --include patterns are specified, the only files that + are processed are those whose names match one of the patterns + and do not match an --exclude pattern. This option does not + affect directories, but it applies to all files, whether + listed on the command line, obtained from --file-list, or by + scanning a directory. The pattern is a PCRE2 regular expres- + sion, and is matched against the final component of the file + name, not the entire path. The -F, -w, and -x options do not + apply to this pattern. The option may be given any number of + times. If a file name matches both an --include and an --ex- + clude pattern, it is excluded. There is no short form for + this option. --include-from=filename - Treat each non-empty line of the file as the data for an + Treat each non-empty line of the file as the data for an --include option. What constitutes a newline for this purpose - is the operating system's default. The --newline option has + is the operating system's default. The --newline option has no effect on this option. This option may be given any number of times; all the files are read. --include-dir=pattern - If any --include-dir patterns are specified, the only direc- - tories that are processed are those that match one of the - patterns (and do not match an --exclude-dir pattern). This - applies to all directories, whether listed on the command - line, obtained from --file-list, or by scanning a parent - directory. The pattern is a PCRE2 regular expression, and is - matched against the final component of the directory name, - not the entire path. The -F, -w, and -x options do not apply + If any --include-dir patterns are specified, the only direc- + tories that are processed are those whose names match one of + the patterns and do not match an --exclude-dir pattern. This + applies to all directories, whether listed on the command + line, obtained from --file-list, or by scanning a parent di- + rectory. The pattern is a PCRE2 regular expression, and is + matched against the final component of the directory name, + not the entire path. The -F, -w, and -x options do not apply to this pattern. The option may be given any number of times. - If a directory matches both --include-dir and --exclude-dir, + If a directory matches both --include-dir and --exclude-dir, it is excluded. There is no short form for this option. -L, --files-without-match - Instead of outputting lines from the files, just output the - names of the files that do not contain any lines that would - have been output. Each file name is output once, on a sepa- - rate line. This option overrides any previous -H, -h, or -l + Instead of outputting lines from the files, just output the + names of the files that do not contain any lines that would + have been output. Each file name is output once, on a sepa- + rate line. This option overrides any previous -H, -h, or -l options. -l, --files-with-matches - Instead of outputting lines from the files, just output the + Instead of outputting lines from the files, just output the names of the files containing lines that would have been out- - put. Each file name is output once, on a separate line. - Searching normally stops as soon as a matching line is found - in a file. However, if the -c (count) option is also used, - matching continues in order to obtain the correct count, and - those files that have at least one match are listed along + put. Each file name is output once, on a separate line. + Searching normally stops as soon as a matching line is found + in a file. However, if the -c (count) option is also used, + matching continues in order to obtain the correct count, and + those files that have at least one match are listed along with their counts. Using this option with -c is a way of sup- - pressing the listing of files with no matches. This opeion - overrides any previous -H, -h, or -L options. + pressing the listing of files with no matches that occurs + with -c on its own. This option overrides any previous -H, + -h, or -L options. --label=name This option supplies a name to be used for the standard input @@ -474,15 +474,15 @@ OPTIONS When this option is given, non-compressed input is read and processed line by line, and the output is flushed after each write. By default, input is read in large chunks, unless - pcre2grep can determine that it is reading from a terminal - (which is currently possible only in Unix-like environments - or Windows). Output to terminal is normally automatically - flushed by the operating system. This option can be useful - when the input or output is attached to a pipe and you do not - want pcre2grep to buffer up large amounts of data. However, - its use will affect performance, and the -M (multiline) - option ceases to work. When input is from a compressed .gz or - .bz2 file, --line-buffered is ignored. + pcre2grep can determine that it is reading from a terminal, + which is currently possible only in Unix-like environments or + Windows. Output to terminal is normally automatically flushed + by the operating system. This option can be useful when the + input or output is attached to a pipe and you do not want + pcre2grep to buffer up large amounts of data. However, its + use will affect performance, and the -M (multiline) option + ceases to work. When input is from a compressed .gz or .bz2 + file, --line-buffered is ignored. --line-offsets Instead of showing lines or parts of lines that match, show @@ -498,134 +498,159 @@ OPTIONS --locale=locale-name This option specifies a locale to be used for pattern match- ing. It overrides the value in the LC_ALL or LC_CTYPE envi- - ronment variables. If no locale is specified, the PCRE2 - library's default (usually the "C" locale) is used. There is - no short form for this option. + ronment variables. If no locale is specified, the PCRE2 li- + brary's default (usually the "C" locale) is used. There is no + short form for this option. + + -M, --multiline + Allow patterns to match more than one line. When this option + is set, the PCRE2 library is called in "multiline" mode. This + allows a matched string to extend past the end of a line and + continue on one or more subsequent lines. Patterns used with + -M may usefully contain literal newline characters and inter- + nal occurrences of ^ and $ characters. The output for a suc- + cessful match may consist of more than one line. The first + line is the line in which the match started, and the last + line is the line in which the match ended. If the matched + string ends with a newline sequence, the output ends at the + end of that line. If -v is set, none of the lines in a + multi-line match are output. Once a match has been handled, + scanning restarts at the beginning of the line after the one + in which the match ended. + + The newline sequence that separates multiple lines must be + matched as part of the pattern. For example, to find the + phrase "regular expression" in a file where "regular" might + be at the end of a line and "expression" at the start of the + next line, you could use this command: + + pcre2grep -M 'regular\s+expression' <file> + + The \s escape sequence matches any white space character, in- + cluding newlines, and is followed by + so as to match trail- + ing white space on the first line as well as possibly han- + dling a two-character newline sequence. + + There is a limit to the number of lines that can be matched, + imposed by the way that pcre2grep buffers the input file as + it scans it. With a sufficiently large processing buffer, + this should not be a problem, but the -M option does not work + when input is read line by line (see --line-buffered.) + + -m number, --max-count=number + Stop processing after finding number matching lines, or non- + matching lines if -v is also set. Any trailing context lines + are output after the final match. In multiline mode, each + multiline match counts as just one line for this purpose. If + this limit is reached when reading the standard input from a + regular file, the file is left positioned just after the last + matching line. If -c is also set, the count that is output + is never greater than number. This option has no effect if + used with -L, -l, or -q, or when just checking for a match in + a binary file. --match-limit=number - Processing some regular expression patterns may take a very + Processing some regular expression patterns may take a very long time to search for all possible matching strings. Others - may require a very large amount of memory. There are three + may require a very large amount of memory. There are three options that set resource limits for matching. The --match-limit option provides a means of limiting comput- - ing resource usage when processing patterns that are not - going to match, but which have a very large number of possi- - bilities in their search trees. The classic example is a pat- - tern that uses nested unlimited repeats. Internally, PCRE2 - has a counter that is incremented each time around its main - processing loop. If the value set by --match-limit is - reached, an error occurs. - - The --heap-limit option specifies, as a number of kibibytes - (units of 1024 bytes), the amount of heap memory that may be + ing resource usage when processing patterns that are not go- + ing to match, but which have a very large number of possibil- + ities in their search trees. The classic example is a pattern + that uses nested unlimited repeats. Internally, PCRE2 has a + counter that is incremented each time around its main pro- + cessing loop. If the value set by --match-limit is reached, + an error occurs. + + The --heap-limit option specifies, as a number of kibibytes + (units of 1024 bytes), the amount of heap memory that may be used for matching. Heap memory is needed only if matching the - pattern requires a significant number of nested backtracking + pattern requires a significant number of nested backtracking points to be remembered. This parameter can be set to zero to forbid the use of heap memory altogether. - The --depth-limit option limits the depth of nested back- + The --depth-limit option limits the depth of nested back- tracking points, which indirectly limits the amount of memory that is used. The amount of memory needed for each backtrack- - ing point depends on the number of capturing parentheses in + ing point depends on the number of capturing parentheses in the pattern, so the amount of memory that is used before this - limit acts varies from pattern to pattern. This limit is of + limit acts varies from pattern to pattern. This limit is of use only if it is set smaller than --match-limit. - There are no short forms for these options. The default lim- - its can be set when the PCRE2 library is compiled; if they - are not specified, the defaults are very large and so effec- + There are no short forms for these options. The default lim- + its can be set when the PCRE2 library is compiled; if they + are not specified, the defaults are very large and so effec- tively unlimited. --max-buffer-size=number - This limits the expansion of the processing buffer, whose - initial size can be set by --buffer-size. The maximum buffer - size is silently forced to be no smaller than the starting + This limits the expansion of the processing buffer, whose + initial size can be set by --buffer-size. The maximum buffer + size is silently forced to be no smaller than the starting buffer size. - -M, --multiline - Allow patterns to match more than one line. When this option - is set, the PCRE2 library is called in "multiline" mode. This - allows a matched string to extend past the end of a line and - continue on one or more subsequent lines. Patterns used with - -M may usefully contain literal newline characters and inter- - nal occurrences of ^ and $ characters. The output for a suc- - cessful match may consist of more than one line. The first - line is the line in which the match started, and the last - line is the line in which the match ended. If the matched - string ends with a newline sequence, the output ends at the - end of that line. If -v is set, none of the lines in a - multi-line match are output. Once a match has been handled, - scanning restarts at the beginning of the line after the one - in which the match ended. - - The newline sequence that separates multiple lines must be - matched as part of the pattern. For example, to find the - phrase "regular expression" in a file where "regular" might - be at the end of a line and "expression" at the start of the - next line, you could use this command: - - pcre2grep -M 'regular\s+expression' <file> - - The \s escape sequence matches any white space character, - including newlines, and is followed by + so as to match - trailing white space on the first line as well as possibly - handling a two-character newline sequence. - - There is a limit to the number of lines that can be matched, - imposed by the way that pcre2grep buffers the input file as - it scans it. With a sufficiently large processing buffer, - this should not be a problem, but the -M option does not work - when input is read line by line (see --line-buffered.) - -N newline-type, --newline=newline-type - The PCRE2 library supports five different conventions for - indicating the ends of lines. They are the single-character - sequences CR (carriage return) and LF (linefeed), the two- - character sequence CRLF, an "anycrlf" convention, which rec- - ognizes any of the preceding three types, and an "any" con- - vention, in which any Unicode line ending sequence is assumed - to end a line. The Unicode sequences are the three just men- - tioned, plus VT (vertical tab, U+000B), FF (form feed, - U+000C), NEL (next line, U+0085), LS (line separator, - U+2028), and PS (paragraph separator, U+2029). - - When the PCRE2 library is built, a default line-ending - sequence is specified. This is normally the standard - sequence for the operating system. Unless otherwise specified - by this option, pcre2grep uses the library's default. The - possible values for this option are CR, LF, CRLF, ANYCRLF, or - ANY. This makes it possible to use pcre2grep to scan files + Six different conventions for indicating the ends of lines in + scanned files are supported. For example: + + pcre2grep -N CRLF 'some pattern' <file> + + The newline type may be specified in upper, lower, or mixed + case. If the newline type is NUL, lines are separated by bi- + nary zero characters. The other types are the single-charac- + ter sequences CR (carriage return) and LF (linefeed), the + two-character sequence CRLF, an "anycrlf" type, which recog- + nizes any of the preceding three types, and an "any" type, + for which any Unicode line ending sequence is assumed to end + a line. The Unicode sequences are the three just mentioned, + plus VT (vertical tab, U+000B), FF (form feed, U+000C), NEL + (next line, U+0085), LS (line separator, U+2028), and PS + (paragraph separator, U+2029). + + When the PCRE2 library is built, a default line-ending se- + quence is specified. This is normally the standard sequence + for the operating system. Unless otherwise specified by this + option, pcre2grep uses the library's default. + + This option makes it possible to use pcre2grep to scan files that have come from other environments without having to mod- - ify their line endings. If the data that is being scanned - does not agree with the convention set by this option, - pcre2grep may behave in strange ways. Note that this option - does not apply to files specified by the -f, --exclude-from, - or --include-from options, which are expected to use the - operating system's standard newline sequence. + ify their line endings. If the data that is being scanned + does not agree with the convention set by this option, + pcre2grep may behave in strange ways. Note that this option + does not apply to files specified by the -f, --exclude-from, + or --include-from options, which are expected to use the op- + erating system's standard newline sequence. -n, --line-number Precede each output line by its line number in the file, fol- - lowed by a colon for matching lines or a hyphen for context + lowed by a colon for matching lines or a hyphen for context lines. If the file name is also being output, it precedes the - line number. When the -M option causes a pattern to match - more than one line, only the first is preceded by its line + line number. When the -M option causes a pattern to match + more than one line, only the first is preceded by its line number. This option is forced if --line-offsets is used. - --no-jit If the PCRE2 library is built with support for just-in-time + --no-jit If the PCRE2 library is built with support for just-in-time compiling (which speeds up matching), pcre2grep automatically makes use of this, unless it was explicitly disabled at build - time. This option can be used to disable the use of JIT at - run time. It is provided for testing and working round prob- + time. This option can be used to disable the use of JIT at + run time. It is provided for testing and working round prob- lems. It should never be needed in normal use. -O text, --output=text - When there is a match, instead of outputting the whole line - that matched, output just the given text. This option is - mutually exclusive with --only-matching, --file-offsets, and - --line-offsets. Escape sequences starting with a dollar char- - acter may be used to insert the contents of the matched part - of the line and/or captured substrings into the text. + When there is a match, instead of outputting the line that + matched, output just the text specified in this option, fol- + lowed by an operating-system standard newline. In this mode, + no context is shown. That is, the -A, -B, and -C options are + ignored. The --newline option has no effect on this option, + which is mutually exclusive with --only-matching, --file-off- + sets, and --line-offsets. However, like --only-matching, if + there is more than one match in a line, each of them causes a + line of output. + + Escape sequences starting with a dollar character may be used + to insert the contents of the matched part of the line and/or + captured substrings into the text. $<digits> or ${<digits>} is replaced by the captured sub- string of the given decimal number; zero substitutes the @@ -637,11 +662,17 @@ OPTIONS form feed; $n by newline; $r by carriage return; $t by tab; $v by vertical tab. - $o<digits> is replaced by the character represented by the - given octal number; up to three digits are processed. + $o<digits> or $o{<digits>} is replaced by the character whose + code point is the given octal number. In the first form, up + to three octal digits are processed. When more digits are + needed in Unicode mode to specify a wide character, the sec- + ond form must be used. - $x<digits> is replaced by the character represented by the - given hexadecimal number; up to two digits are processed. + $x<digits> or $x{<digits>} is replaced by the character rep- + resented by the given hexadecimal number. In the first form, + up to two hexadecimal digits are processed. When more digits + are needed in Unicode mode to specify a wide character, the + second form must be used. Any other character is substituted by itself. In particular, $$ is replaced by a single dollar. @@ -651,9 +682,9 @@ OPTIONS of the whole line. In this mode, no context is shown. That is, the -A, -B, and -C options are ignored. If there is more than one match in a line, each of them is shown separately, - on a separate line of output. If -o is combined with -v - (invert the sense of the match to find non-matching lines), - no output is generated, but the return code is set appropri- + on a separate line of output. If -o is combined with -v (in- + vert the sense of the match to find non-matching lines), no + output is generated, but the return code is set appropri- ately. If the matched portion of the line is empty, nothing is output unless the file name or line number are being printed, in which case they are shown on an otherwise empty @@ -662,23 +693,32 @@ OPTIONS -onumber, --only-matching=number Show only the part of the line that matched the capturing - parentheses of the given number. Up to 32 capturing parenthe- - ses are supported, and -o0 is equivalent to -o without a num- - ber. Because these options can be given without an argument - (see above), if an argument is present, it must be given in - the same shell item, for example, -o3 or --only-matching=2. - The comments given for the non-argument case above also apply - to this option. If the specified capturing parentheses do not - exist in the pattern, or were not set in the match, nothing - is output unless the file name or line number are being out- - put. - - If this option is given multiple times, multiple substrings - are output for each match, in the order the options are - given, and all on one line. For example, -o3 -o1 -o3 causes - the substrings matched by capturing parentheses 3 and 1 and - then 3 again to be output. By default, there is no separator - (but see the next option). + parentheses of the given number. Up to 50 capturing parenthe- + ses are supported by default. This limit can be changed via + the --om-capture option. A pattern may contain any number of + capturing parentheses, but only those whose number is within + the limit can be accessed by -o. An error occurs if the num- + ber specified by -o is greater than the limit. + + -o0 is the same as -o without a number. Because these options + can be given without an argument (see above), if an argument + is present, it must be given in the same shell item, for ex- + ample, -o3 or --only-matching=2. The comments given for the + non-argument case above also apply to this option. If the + specified capturing parentheses do not exist in the pattern, + or were not set in the match, nothing is output unless the + file name or line number are being output. + + If this option is given multiple times, multiple substrings + are output for each match, in the order the options are + given, and all on one line. For example, -o3 -o1 -o3 causes + the substrings matched by capturing parentheses 3 and 1 and + then 3 again to be output. By default, there is no separator + (but see the next but one option). + + --om-capture=number + Set the number of capturing parentheses that can be accessed + by -o. The default is 50. --om-separator=text Specify a separating string for multiple occurrences of -o. @@ -695,45 +735,57 @@ OPTIONS it contains, taking note of any --include and --exclude set- tings. By default, a directory is read as a normal file; in some operating systems this gives an immediate end-of-file. - This option is a shorthand for setting the -d option to - "recurse". + This option is a shorthand for setting the -d option to "re- + curse". --recursion-limit=number - See --match-limit above. + This is an obsolete synonym for --depth-limit. See --match- + limit above for details. -s, --no-messages - Suppress error messages about non-existent or unreadable - files. Such files are quietly skipped. However, the return + Suppress error messages about non-existent or unreadable + files. Such files are quietly skipped. However, the return code is still 2, even if matches were found in other files. -t, --total-count - This option is useful when scanning more than one file. If - used on its own, -t suppresses all output except for a grand - total number of matching lines (or non-matching lines if -v - is used) in all the files. If -t is used with -c, a grand - total is output except when the previous output is just one - line. In other words, it is not output when just one file's - count is listed. If file names are being output, the grand - total is preceded by "TOTAL:". Otherwise, it appears as just - another number. The -t option is ignored when used with -L - (list files without matches), because the grand total would + This option is useful when scanning more than one file. If + used on its own, -t suppresses all output except for a grand + total number of matching lines (or non-matching lines if -v + is used) in all the files. If -t is used with -c, a grand to- + tal is output except when the previous output is just one + line. In other words, it is not output when just one file's + count is listed. If file names are being output, the grand + total is preceded by "TOTAL:". Otherwise, it appears as just + another number. The -t option is ignored when used with -L + (list files without matches), because the grand total would always be zero. - -u, --utf-8 - Operate in UTF-8 mode. This option is available only if PCRE2 + -u, --utf Operate in UTF-8 mode. This option is available only if PCRE2 has been compiled with UTF-8 support. All patterns (including - those for any --exclude and --include options) and all sub- - ject lines that are scanned must be valid strings of UTF-8 - characters. + those for any --exclude and --include options) and all lines + that are scanned must be valid strings of UTF-8 characters. + If an invalid UTF-8 string is encountered, an error occurs. + + -U, --utf-allow-invalid + As --utf, but in addition subject lines may contain invalid + UTF-8 code unit sequences. These can never form part of any + pattern match. Patterns themselves, however, must still be + valid UTF-8 strings. This facility allows valid UTF-8 strings + to be sought within arbitrary byte sequences in executable or + other binary files. For more details about matching in non- + valid UTF-8 strings, see the pcre2unicode(3) documentation. -V, --version - Write the version numbers of pcre2grep and the PCRE2 library - to the standard output and then exit. Anything else on the + Write the version numbers of pcre2grep and the PCRE2 library + to the standard output and then exit. Anything else on the command line is ignored. -v, --invert-match - Invert the sense of the match, so that lines which do not - match any of the patterns are the ones that are found. + Invert the sense of the match, so that lines which do not + match any of the patterns are the ones that are found. When + this option is set, options such as --only-matching and + --output, which specify parts of a match that are to be out- + put, are ignored. -w, --word-regex, --word-regexp Force the patterns only to match "words". That is, there must @@ -757,25 +809,35 @@ OPTIONS ENVIRONMENT VARIABLES - The environment variables LC_ALL and LC_CTYPE are examined, in that - order, for a locale. The first one that is set is used. This can be - overridden by the --locale option. If no locale is set, the PCRE2 - library's default (usually the "C" locale) is used. + The environment variables LC_ALL and LC_CTYPE are examined, in that or- + der, for a locale. The first one that is set is used. This can be over- + ridden by the --locale option. If no locale is set, the PCRE2 library's + default (usually the "C" locale) is used. NEWLINES - The -N (--newline) option allows pcre2grep to scan files with different - newline conventions from the default. Any parts of the input files that - are written to the standard output are copied identically, with what- - ever newline sequences they have in the input. However, the setting of - this option affects only the way scanned files are processed. It does - not affect the interpretation of files specified by the -f, --file- - list, --exclude-from, or --include-from options, nor does it affect the - way in which pcre2grep writes informational messages to the standard - error and output streams. For these it uses the string "\n" to indicate - newlines, relying on the C I/O library to convert this to an appropri- - ate sequence. + The -N (--newline) option allows pcre2grep to scan files with newline + conventions that differ from the default. This option affects only the + way scanned files are processed. It does not affect the interpretation + of files specified by the -f, --file-list, --exclude-from, or --in- + clude-from options. + + Any parts of the scanned input files that are written to the standard + output are copied with whatever newline sequences they have in the in- + put. However, if the final line of a file is output, and it does not + end with a newline sequence, a newline sequence is added. If the new- + line setting is CR, LF, CRLF or NUL, that line ending is output; for + the other settings (ANYCRLF or ANY) a single NL is used. + + The newline setting does not affect the way in which pcre2grep writes + newlines in informational messages to the standard output and error + streams. Under Windows, the standard output is set to be binary, so + that "\r\n" at the ends of output lines that are copied from the input + is not converted to "\r\r\n" by the C I/O library. This means that any + messages written to the standard output must end with "\r\n". For all + other operating systems, and for all messages to the standard error + stream, "\n" is used. OPTIONS COMPATIBILITY @@ -785,9 +847,9 @@ OPTIONS COMPATIBILITY terminology) is also available as --xxx-regex (PCRE2 terminology). How- ever, the --depth-limit, --file-list, --file-offsets, --heap-limit, --include-dir, --line-offsets, --locale, --match-limit, -M, --multi- - line, -N, --newline, --om-separator, --output, -u, and --utf-8 options - are specific to pcre2grep, as is the use of the --only-matching option - with a capturing parentheses number. + line, -N, --newline, --om-separator, --output, -u, --utf, -U, and + --utf-allow-invalid options are specific to pcre2grep, as is the use of + the --only-matching option with a capturing parentheses number. Although most of the common options work the same way, a few are dif- ferent in pcre2grep. For example, the --include option's argument is a @@ -818,13 +880,13 @@ OPTIONS WITH DATA --file /some/file Note, however, that if you want to supply a file name beginning with ~ - as data in a shell command, and have the shell expand ~ to a home - directory, you must separate the file name from the option, because the + as data in a shell command, and have the shell expand ~ to a home di- + rectory, you must separate the file name from the option, because the shell does not treat ~ specially unless it is at the start of an item. The exceptions to the above are the --colour (or --color) and --only- - matching options, for which the data is optional. If one of these - options does have data, it must be given in the first form, using an + matching options, for which the data is optional. If one of these op- + tions does have data, it must be given in the first form, using an equals character. Otherwise pcre2grep will assume that it has no data. @@ -834,8 +896,8 @@ USING PCRE2'S CALLOUT FACILITY scripts or echoing specific strings during matching by making use of PCRE2's callout facility. However, this support can be completely or partially disabled when pcre2grep is built. You can find out whether - your binary has support for callouts by running it with the --help - option. If callout support is completely disabled, all callouts in pat- + your binary has support for callouts by running it with the --help op- + tion. If callout support is completely disabled, all callouts in pat- terns are ignored by pcre2grep. If the facility is partially disabled, calling external programs is not supported, and callouts that request it are ignored. @@ -845,30 +907,49 @@ USING PCRE2'S CALLOUT FACILITY mentation for details). Numbered callouts are ignored by pcre2grep; only callouts with string arguments are useful. + Echoing a specific string + + Starting the callout string with a pipe character invokes an echoing + facility that avoids calling an external program or script. This facil- + ity is always available, provided that callouts were not completely + disabled when pcre2grep was built. The rest of the callout string is + processed as a zero-terminated string, which means it should not con- + tain any internal binary zeros. It is written to the output, having + first been passed through the same escape processing as text from the + --output (-O) option (see above). However, $0 cannot be used to insert + a matched substring because the match is still in progress. Instead, + the single character '0' is inserted. Any syntax errors in the string + (for example, a dollar not followed by another character) causes the + callout to be ignored. No terminator is added to the output string, so + if you want a newline, you must include it explicitly using the escape + $n. For example: + + pcre2grep '(.)(..(.))(?C"|[$1] [$2] [$3]$n")' <some file> + + Matching continues normally after the string is output. If you want to + see only the callout output but not any output from an actual match, + you should end the pattern with (*FAIL). + Calling external programs or scripts This facility can be independently disabled when pcre2grep is built. It - is supported for Windows, where a call to _spawnvp() is used, for VMS, - where lib$spawn() is used, and for any other Unix-like environment - where fork() and execv() are available. + is supported for Windows, where a call to _spawnvp() is used, for VMS, + where lib$spawn() is used, and for any Unix-like environment where + fork() and execv() are available. If the callout string does not start with a pipe (vertical bar) charac- - ter, it is parsed into a list of substrings separated by pipe charac- - ters. The first substring must be an executable name, with the follow- + ter, it is parsed into a list of substrings separated by pipe charac- + ters. The first substring must be an executable name, with the follow- ing substrings specifying arguments: executable_name|arg1|arg2|... - Any substring (including the executable name) may contain escape - sequences started by a dollar character: $<digits> or ${<digits>} is - replaced by the captured substring of the given decimal number, which - must be greater than zero. If the number is greater than the number of - capturing substrings, or if the capture is unset, the replacement is - empty. - - Any other character is substituted by itself. In particular, $$ is - replaced by a single dollar and $| is replaced by a pipe character. - Here is an example: + Any substring (including the executable name) may contain escape se- + quences started by a dollar character. These are the same as for the + --output (-O) option documented above, except that $0 cannot insert the + matched string because the match is still in progress. Instead, the + character '0' is inserted. If you need a literal dollar or pipe charac- + ter in any substring, use $$ or $| respectively. Here is an example: echo -e "abcde\n12345" | pcre2grep \ '(?x)(.)(..(.)) @@ -881,36 +962,23 @@ USING PCRE2'S CALLOUT FACILITY Arg1: [1] [234] [4] Arg2: |1| () 12345 - The parameters for the system call that is used to run the program or + The parameters for the system call that is used to run the program or script are zero-terminated strings. This means that binary zero charac- - ters in the callout argument will cause premature termination of their - substrings, and therefore should not be present. Any syntax errors in - the string (for example, a dollar not followed by another character) - cause the callout to be ignored. If running the program fails for any - reason (including the non-existence of the executable), a local match- + ters in the callout argument will cause premature termination of their + substrings, and therefore should not be present. Any syntax errors in + the string (for example, a dollar not followed by another character) + causes the callout to be ignored. If running the program fails for any + reason (including the non-existence of the executable), a local match- ing failure occurs and the matcher backtracks in the normal way. - Echoing a specific string - - This facility is always available, provided that callouts were not com- - pletely disabled when pcre2grep was built. If the callout string starts - with a pipe (vertical bar) character, the rest of the string is written - to the output, having been passed through the same escape processing as - text from the --output option. This provides a simple echoing facility - that avoids calling an external program or script. No terminator is - added to the string, so if you want a newline, you must include it - explicitly. Matching continues normally after the string is output. If - you want to see only the callout output but not any output from an - actual match, you should end the relevant pattern with (*FAIL). - MATCHING ERRORS It is possible to supply a regular expression that takes a very long time to fail to match certain lines. Such patterns normally involve nested indefinite repeats, for example: (a+)*\d when matched against a - line of a's with no final digit. The PCRE2 matching function has a - resource limit that causes it to abort in these circumstances. If this + line of a's with no final digit. The PCRE2 matching function has a re- + source limit that causes it to abort in these circumstances. If this happens, pcre2grep outputs an error message and the line that caused the problem to the standard error stream. If there are more than 20 such errors, pcre2grep gives up. @@ -936,7 +1004,7 @@ DIAGNOSTICS SEE ALSO - pcre2pattern(3), pcre2syntax(3), pcre2callout(3). + pcre2pattern(3), pcre2syntax(3), pcre2callout(3), pcre2unicode(3). AUTHOR @@ -948,5 +1016,5 @@ AUTHOR REVISION - Last updated: 24 November 2018 - Copyright (c) 1997-2018 University of Cambridge. + Last updated: 04 October 2020 + Copyright (c) 1997-2020 University of Cambridge. diff --git a/dist2/doc/pcre2jit.3 b/dist2/doc/pcre2jit.3 index b7ae2eb1..fab83666 100644 --- a/dist2/doc/pcre2jit.3 +++ b/dist2/doc/pcre2jit.3 @@ -1,4 +1,4 @@ -.TH PCRE2JIT 3 "06 March 2019" "PCRE2 10.33" +.TH PCRE2JIT 3 "23 May 2019" "PCRE2 10.34" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH "PCRE2 JUST-IN-TIME COMPILER SUPPORT" @@ -64,7 +64,7 @@ or a negative error code. There is a limit to the size of pattern that JIT supports, imposed by the size of machine stack that it uses. The exact rules are not documented because they may change at any time, in particular, when new optimizations are introduced. -If a pattern is too big, a call to \fBpcre2_jit_compile()\fB returns +If a pattern is too big, a call to \fBpcre2_jit_compile()\fP returns PCRE2_ERROR_NOMEMORY. .P PCRE2_JIT_COMPLETE requests the JIT compiler to generate code for complete @@ -123,23 +123,29 @@ pattern. .SH "MATCHING SUBJECTS CONTAINING INVALID UTF" .rs .sp -When a pattern is compiled with the PCRE2_UTF option, the interpretive matching -function expects its subject string to be a valid sequence of UTF code units. -If it is not, the result is undefined. This is also true by default of matching -via JIT. However, if the option PCRE2_JIT_INVALID_UTF is passed to -\fBpcre2_jit_compile()\fP, code that can process a subject containing invalid -UTF is compiled. +When a pattern is compiled with the PCRE2_UTF option, subject strings are +normally expected to be a valid sequence of UTF code units. By default, this is +checked at the start of matching and an error is generated if invalid UTF is +detected. The PCRE2_NO_UTF_CHECK option can be passed to \fBpcre2_match()\fP to +skip the check (for improved performance) if you are sure that a subject string +is valid. If this option is used with an invalid string, the result is +undefined. .P -In this mode, an invalid code unit sequence never matches any pattern item. It -does not match dot, it does not match \ep{Any}, it does not even match negative -items such as [^X]. A lookbehind assertion fails if it encounters an invalid -sequence while moving the current point backwards. In other words, an invalid -UTF code unit sequence acts as a barrier which no match can cross. Reaching an -invalid sequence causes an immediate backtrack. +However, a way of running matches on strings that may contain invalid UTF +sequences is available. Calling \fBpcre2_compile()\fP with the +PCRE2_MATCH_INVALID_UTF option has two effects: it tells the interpreter in +\fBpcre2_match()\fP to support invalid UTF, and, if \fBpcre2_jit_compile()\fP +is called, the compiled JIT code also supports invalid UTF. Details of how this +support works, in both the JIT and the interpretive cases, is given in the +.\" HREF +\fBpcre2unicode\fP +.\" +documentation. .P -Using this option, an application can run matches in arbitrary data, knowing -that any matched strings that are returned will be valid UTF. This can be -useful when searching for text in executable or other binary files. +There is also an obsolete option for \fBpcre2_jit_compile()\fP called +PCRE2_JIT_INVALID_UTF, which currently exists only for backward compatibility. +It is superseded by the \fBpcre2_compile()\fP option PCRE2_MATCH_INVALID_UTF +and should no longer be used. It may be removed in future. . . .SH "UNSUPPORTED OPTIONS AND PATTERN ITEMS" @@ -309,12 +315,12 @@ stack through the JIT callback function. You can free a JIT stack at any time, as long as it will not be used by \fBpcre2_match()\fP again. When you assign the stack to a match context, only a pointer is set. There is no reference counting or any other magic. You can free -compiled patterns, contexts, and stacks in any order, anytime. Just \fIdo -not\fP call \fBpcre2_match()\fP with a match context pointing to an already -freed stack, as that will cause SEGFAULT. (Also, do not free a stack currently -used by \fBpcre2_match()\fP in another thread). You can also replace the stack -in a context at any time when it is not in use. You should free the previous -stack before assigning a replacement. +compiled patterns, contexts, and stacks in any order, anytime. +Just \fIdo not\fP call \fBpcre2_match()\fP with a match context pointing to an +already freed stack, as that will cause SEGFAULT. (Also, do not free a stack +currently used by \fBpcre2_match()\fP in another thread). You can also replace +the stack in a context at any time when it is not in use. You should free the +previous stack before assigning a replacement. .P (5) Should I allocate/free a stack every time before/after calling \fBpcre2_match()\fP? @@ -438,6 +444,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 06 March 2019 +Last updated: 23 May 2019 Copyright (c) 1997-2019 University of Cambridge. .fi diff --git a/dist2/doc/pcre2matching.3 b/dist2/doc/pcre2matching.3 index 20ffac5b..7f9bbac7 100644 --- a/dist2/doc/pcre2matching.3 +++ b/dist2/doc/pcre2matching.3 @@ -1,4 +1,4 @@ -.TH PCRE2MATCHING 3 "10 October 2018" "PCRE2 10.33" +.TH PCRE2MATCHING 3 "23 May 2019" "PCRE2 10.34" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH "PCRE2 MATCHING ALGORITHMS" @@ -157,6 +157,9 @@ code unit) at a time, for all active paths through the tree. .P 9. Except for (*FAIL), the backtracking control verbs such as (*PRUNE) are not supported. (*FAIL) is supported, and behaves like a failing negative assertion. +.P +10. The PCRE2_MATCH_INVALID_UTF option for \fBpcre2_compile()\fP is not +supported by \fBpcre2_dfa_match()\fP. . . .SH "ADVANTAGES OF THE ALTERNATIVE ALGORITHM" @@ -191,7 +194,8 @@ The alternative algorithm suffers from a number of disadvantages: because it has to search for all possible matches, but is also because it is less susceptible to optimization. .P -2. Capturing parentheses, backreferences, and script runs are not supported. +2. Capturing parentheses, backreferences, script runs, and matching within +invalid UTF string are not supported. .P 3. Although atomic groups are supported, their use does not provide the performance advantage that it does for the standard algorithm. @@ -211,6 +215,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 10 October 2018 -Copyright (c) 1997-2018 University of Cambridge. +Last updated: 23 May 2019 +Copyright (c) 1997-2019 University of Cambridge. .fi diff --git a/dist2/doc/pcre2partial.3 b/dist2/doc/pcre2partial.3 index 72a12ed2..892906a7 100644 --- a/dist2/doc/pcre2partial.3 +++ b/dist2/doc/pcre2partial.3 @@ -1,67 +1,107 @@ -.TH PCRE2PARTIAL 3 "22 December 2014" "PCRE2 10.00" +.TH PCRE2PARTIAL 3 "04 September 2019" "PCRE2 10.34" .SH NAME PCRE2 - Perl-compatible regular expressions .SH "PARTIAL MATCHING IN PCRE2" .rs .sp -In normal use of PCRE2, if the subject string that is passed to a matching -function matches as far as it goes, but is too short to match the entire -pattern, PCRE2_ERROR_NOMATCH is returned. There are circumstances where it -might be helpful to distinguish this case from other cases in which there is no -match. +In normal use of PCRE2, if there is a match up to the end of a subject string, +but more characters are needed to match the entire pattern, PCRE2_ERROR_NOMATCH +is returned, just like any other failing match. There are circumstances where +it might be helpful to distinguish this "partial match" case. .P -Consider, for example, an application where a human is required to type in data -for a field with specific formatting requirements. An example might be a date -in the form \fIddmmmyy\fP, defined by this pattern: -.sp - ^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$ -.sp -If the application sees the user's keystrokes one by one, and can check that -what has been typed so far is potentially valid, it is able to raise an error -as soon as a mistake is made, by beeping and not reflecting the character that -has been typed, for example. This immediate feedback is likely to be a better -user interface than a check that is delayed until the entire string has been -entered. Partial matching can also be useful when the subject string is very -long and is not all available at once. +One example is an application where the subject string is very long, and not +all available at once. The requirement here is to be able to do the matching +segment by segment, but special action is needed when a matched substring spans +the boundary between two segments. .P -PCRE2 supports partial matching by means of the PCRE2_PARTIAL_SOFT and -PCRE2_PARTIAL_HARD options, which can be set when calling a matching function. -The difference between the two options is whether or not a partial match is -preferred to an alternative complete match, though the details differ between -the two types of matching function. If both options are set, PCRE2_PARTIAL_HARD -takes precedence. +Another example is checking a user input string as it is typed, to ensure that +it conforms to a required format. Invalid characters can be immediately +diagnosed and rejected, giving instant feedback. .P -If you want to use partial matching with just-in-time optimized code, you must -call \fBpcre2_jit_compile()\fP with one or both of these options: +Partial matching is a PCRE2-specific feature; it is not Perl-compatible. It is +requested by setting one of the PCRE2_PARTIAL_HARD or PCRE2_PARTIAL_SOFT +options when calling a matching function. The difference between the two +options is whether or not a partial match is preferred to an alternative +complete match, though the details differ between the two types of matching +function. If both options are set, PCRE2_PARTIAL_HARD takes precedence. +.P +If you want to use partial matching with just-in-time optimized code, as well +as setting a partial match option for the matching function, you must also call +\fBpcre2_jit_compile()\fP with one or both of these options: .sp - PCRE2_JIT_PARTIAL_SOFT PCRE2_JIT_PARTIAL_HARD + PCRE2_JIT_PARTIAL_SOFT .sp PCRE2_JIT_COMPLETE should also be set if you are going to run non-partial -matches on the same pattern. If the appropriate JIT mode has not been compiled, -interpretive matching code is used. +matches on the same pattern. Separate code is compiled for each mode. If the +appropriate JIT mode has not been compiled, interpretive matching code is used. .P Setting a partial matching option disables two of PCRE2's standard -optimizations. PCRE2 remembers the last literal code unit in a pattern, and -abandons matching immediately if it is not present in the subject string. This -optimization cannot be used for a subject string that might match only -partially. PCRE2 also knows the minimum length of a matching string, and does +optimization hints. PCRE2 remembers the last literal code unit in a pattern, +and abandons matching immediately if it is not present in the subject string. +This optimization cannot be used for a subject string that might match only +partially. PCRE2 also remembers a minimum length of a matching string, and does not bother to run the matching function on shorter strings. This optimization is also disabled for partial matching. . . +.SH "REQUIREMENTS FOR A PARTIAL MATCH" +.rs +.sp +A possible partial match occurs during matching when the end of the subject +string is reached successfully, but either more characters are needed to +complete the match, or the addition of more characters might change what is +matched. +.P +Example 1: if the pattern is /abc/ and the subject is "ab", more characters are +definitely needed to complete a match. In this case both hard and soft matching +options yield a partial match. +.P +Example 2: if the pattern is /ab+/ and the subject is "ab", a complete match +can be found, but the addition of more characters might change what is +matched. In this case, only PCRE2_PARTIAL_HARD returns a partial match; +PCRE2_PARTIAL_SOFT returns the complete match. +.P +On reaching the end of the subject, when PCRE2_PARTIAL_HARD is set, if the next +pattern item is \ez, \eZ, \eb, \eB, or $ there is always a partial match. +Otherwise, for both options, the next pattern item must be one that inspects a +character, and at least one of the following must be true: +.P +(1) At least one character has already been inspected. An inspected character +need not form part of the final matched string; lookbehind assertions and the +\eK escape sequence provide ways of inspecting characters before the start of a +matched string. +.P +(2) The pattern contains one or more lookbehind assertions. This condition +exists in case there is a lookbehind that inspects characters before the start +of the match. +.P +(3) There is a special case when the whole pattern can match an empty string. +When the starting point is at the end of the subject, the empty string match is +a possibility, and if PCRE2_PARTIAL_SOFT is set and neither of the above +conditions is true, it is returned. However, because adding more characters +might result in a non-empty match, PCRE2_PARTIAL_HARD returns a partial match, +which in this case means "there is going to be a match at this point, but until +some more characters are added, we do not know if it will be an empty string or +something longer". +. +. +. .SH "PARTIAL MATCHING USING pcre2_match()" .rs .sp -A partial match occurs during a call to \fBpcre2_match()\fP when the end of the -subject string is reached successfully, but matching cannot continue because -more characters are needed. However, at least one character in the subject must -have been inspected. This character need not form part of the final matched -string; lookbehind assertions and the \eK escape sequence provide ways of -inspecting characters before the start of a matched string. The requirement for -inspecting at least one character exists because an empty string can always be -matched; without such a restriction there would always be a partial match of an -empty string at the end of the subject. +When a partial matching option is set, the result of calling +\fBpcre2_match()\fP can be one of the following: +.TP 2 +\fBA successful match\fP +A complete match has been found, starting and ending within this subject. +.TP +\fBPCRE2_ERROR_NOMATCH\fP +No match can start anywhere in this subject. +.TP +\fBPCRE2_ERROR_PARTIAL\fP +Adding more characters may result in a complete match that uses one or more +characters from the end of this subject. .P When a partial match is returned, the first two elements in the ovector point to the portion of the subject that was matched, but the values in the rest of @@ -77,53 +117,40 @@ is "456abc12", a partial match is found for the string "abc12", because all these characters are needed for a subsequent re-match with additional characters. .P -What happens when a partial match is identified depends on which of the two -partial matching options are set. -. -. -.SS "PCRE2_PARTIAL_SOFT WITH pcre2_match()" -.rs -.sp -If PCRE2_PARTIAL_SOFT is set when \fBpcre2_match()\fP identifies a partial -match, the partial match is remembered, but matching continues as normal, and -other alternatives in the pattern are tried. If no complete match can be found, -PCRE2_ERROR_PARTIAL is returned instead of PCRE2_ERROR_NOMATCH. -.P -This option is "soft" because it prefers a complete match over a partial match. -All the various matching items in a pattern behave as if the subject string is -potentially complete. For example, \ez, \eZ, and $ match at the end of the -subject, as normal, and for \eb and \eB the end of the subject is treated as a -non-alphanumeric. -.P If there is more than one partial match, the first one that was found provides the data that is returned. Consider this pattern: .sp /123\ew+X|dogY/ .sp -If this is matched against the subject string "abc123dog", both -alternatives fail to match, but the end of the subject is reached during -matching, so PCRE2_ERROR_PARTIAL is returned. The offsets are set to 3 and 9, -identifying "123dog" as the first partial match that was found. (In this -example, there are two partial matches, because "dog" on its own partially -matches the second alternative.) +If this is matched against the subject string "abc123dog", both alternatives +fail to match, but the end of the subject is reached during matching, so +PCRE2_ERROR_PARTIAL is returned. The offsets are set to 3 and 9, identifying +"123dog" as the first partial match. (In this example, there are two partial +matches, because "dog" on its own partially matches the second alternative.) . . -.SS "PCRE2_PARTIAL_HARD WITH pcre2_match()" -.rs -.sp -If PCRE2_PARTIAL_HARD is set for \fBpcre2_match()\fP, PCRE2_ERROR_PARTIAL is -returned as soon as a partial match is found, without continuing to search for -possible complete matches. This option is "hard" because it prefers an earlier -partial match over a later complete match. For this reason, the assumption is -made that the end of the supplied subject string may not be the true end of the -available data, and so, if \ez, \eZ, \eb, \eB, or $ are encountered at the end -of the subject, the result is PCRE2_ERROR_PARTIAL, provided that at least one -character in the subject has been inspected. -. -. -.SS "Comparing hard and soft partial matching" +.SS "How a partial match is processed by pcre2_match()" .rs .sp +What happens when a partial match is identified depends on which of the two +partial matching options is set. +.P +If PCRE2_PARTIAL_HARD is set, PCRE2_ERROR_PARTIAL is returned as soon as a +partial match is found, without continuing to search for possible complete +matches. This option is "hard" because it prefers an earlier partial match over +a later complete match. For this reason, the assumption is made that the end of +the supplied subject string is not the true end of the available data, which is +why \ez, \eZ, \eb, \eB, and $ always give a partial match. +.P +If PCRE2_PARTIAL_SOFT is set, the partial match is remembered, but matching +continues as normal, and other alternatives in the pattern are tried. If no +complete match can be found, PCRE2_ERROR_PARTIAL is returned instead of +PCRE2_ERROR_NOMATCH. This option is "soft" because it prefers a complete match +over a partial match. All the various matching items in a pattern behave as if +the subject string is potentially complete; \ez, \eZ, and $ match at the end of +the subject, as normal, and for \eb and \eB the end of the subject is treated +as a non-alphanumeric. +.P The difference between the two partial matching options can be illustrated by a pattern such as: .sp @@ -148,89 +175,151 @@ The second pattern will never match "dogsbody", because it will always find the shorter match first. . . -.SH "PARTIAL MATCHING USING pcre2_dfa_match()" +.SS "Example of partial matching using pcre2test" .rs .sp -The DFA functions move along the subject string character by character, without -backtracking, searching for all possible matches simultaneously. If the end of -the subject is reached before the end of the pattern, there is the possibility -of a partial match, again provided that at least one character has been -inspected. -.P -When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if there -have been no complete matches. Otherwise, the complete matches are returned. -However, if PCRE2_PARTIAL_HARD is set, a partial match takes precedence over -any complete matches. The portion of the string that was matched when the -longest partial match was found is set as the first matching string. -.P -Because the DFA functions always search for all possible matches, and there is -no difference between greedy and ungreedy repetition, their behaviour is -different from the standard functions when PCRE2_PARTIAL_HARD is set. Consider -the string "dog" matched against the ungreedy pattern shown above: +The \fBpcre2test\fP data modifiers \fBpartial_hard\fP (or \fBph\fP) and +\fBpartial_soft\fP (or \fBps\fP) set PCRE2_PARTIAL_HARD and PCRE2_PARTIAL_SOFT, +respectively, when calling \fBpcre2_match()\fP. Here is a run of +\fBpcre2test\fP using a pattern that matches the whole subject in the form of a +date: .sp - /dog(sbody)??/ + re> /^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$/ + data> 25dec3\e=ph + Partial match: 23dec3 + data> 3ju\e=ph + Partial match: 3ju + data> 3juj\e=ph + No match .sp -Whereas the standard function stops as soon as it finds the complete match for -"dog", the DFA function also finds the partial match for "dogsbody", and so -returns that when PCRE2_PARTIAL_HARD is set. +This example gives the same results for both hard and soft partial matching +options. Here is an example where there is a difference: +.sp + re> /^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$/ + data> 25jun04\e=ps + 0: 25jun04 + 1: jun + data> 25jun04\e=ph + Partial match: 25jun04 +.sp +With PCRE2_PARTIAL_SOFT, the subject is matched completely. For +PCRE2_PARTIAL_HARD, however, the subject is assumed not to be complete, so +there is only a partial match. +. . . -.SH "PARTIAL MATCHING AND WORD BOUNDARIES" +.SH "MULTI-SEGMENT MATCHING WITH pcre2_match()" .rs .sp -If a pattern ends with one of sequences \eb or \eB, which test for word -boundaries, partial matching with PCRE2_PARTIAL_SOFT can give counter-intuitive -results. Consider this pattern: +PCRE was not originally designed with multi-segment matching in mind. However, +over time, features (including partial matching) that make multi-segment +matching possible have been added. A very long string can be searched segment +by segment by calling \fBpcre2_match()\fP repeatedly, with the aim of achieving +the same results that would happen if the entire string was available for +searching all the time. Normally, the strings that are being sought are much +shorter than each individual segment, and are in the middle of very long +strings, so the pattern is normally not anchored. +.P +Special logic must be implemented to handle a matched substring that spans a +segment boundary. PCRE2_PARTIAL_HARD should be used, because it returns a +partial match at the end of a segment whenever there is the possibility of +changing the match by adding more characters. The PCRE2_NOTBOL option should +also be set for all but the first segment. +.P +When a partial match occurs, the next segment must be added to the current +subject and the match re-run, using the \fIstartoffset\fP argument of +\fBpcre2_match()\fP to begin at the point where the partial match started. +For example: .sp - /\ebcat\eb/ + re> /\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed/ + data> ...the date is 23ja\e=ph + Partial match: 23ja + data> ...the date is 23jan19 and on that day...\e=offset=15 + 0: 23jan19 + 1: jan +.sp +Note the use of the \fBoffset\fP modifier to start the new match where the +partial match was found. In this example, the next segment was added to the one +in which the partial match was found. This is the most straightforward +approach, typically using a memory buffer that is twice the size of each +segment. After a partial match, the first half of the buffer is discarded, the +second half is moved to the start of the buffer, and a new segment is added +before repeating the match as in the example above. After a no match, the +entire buffer can be discarded. +.P +If there are memory constraints, you may want to discard text that precedes a +partial match before adding the next segment. Unfortunately, this is not at +present straightforward. In cases such as the above, where the pattern does not +contain any lookbehinds, it is sufficient to retain only the partially matched +substring. However, if the pattern contains a lookbehind assertion, characters +that precede the start of the partial match may have been inspected during the +matching process. When \fBpcre2test\fP displays a partial match, it indicates +these characters with '<' if the \fBallusedtext\fP modifier is set: .sp -This matches "cat", provided there is a word boundary at either end. If the -subject string is "the cat", the comparison of the final "t" with a following -character cannot take place, so a partial match is found. However, normal -matching carries on, and \eb matches at the end of the subject when the last -character is a letter, so a complete match is found. The result, therefore, is -\fInot\fP PCRE2_ERROR_PARTIAL. Using PCRE2_PARTIAL_HARD in this case does yield -PCRE2_ERROR_PARTIAL, because then the partial match takes precedence. + re> "(?<=123)abc" + data> xx123ab\e=ph,allusedtext + Partial match: 123ab + <<< +.sp +However, the \fBallusedtext\fP modifier is not available for JIT matching, +because JIT matching does not record the first (or last) consulted characters. +For this reason, this information is not available via the API. It is therefore +not possible in general to obtain the exact number of characters that must be +retained in order to get the right match result. If you cannot retain the +entire segment, you must find some heuristic way of choosing. +.P +If you know the approximate length of the matching substrings, you can use that +to decide how much text to retain. The only lookbehind information that is +currently available via the API is the length of the longest individual +lookbehind in a pattern, but this can be misleading if there are nested +lookbehinds. The value returned by calling \fBpcre2_pattern_info()\fP with the +PCRE2_INFO_MAXLOOKBEHIND option is the maximum number of characters (not code +units) that any individual lookbehind moves back when it is processed. A +pattern such as "(?<=(?<!b)a)" has a maximum lookbehind value of one, but +inspects two characters before its starting point. +.P +In a non-UTF or a 32-bit case, moving back is just a subtraction, but in +UTF-8 or UTF-16 you have to count characters while moving back through the code +units. . . -.SH "EXAMPLE OF PARTIAL MATCHING USING PCRE2TEST" +.SH "PARTIAL MATCHING USING pcre2_dfa_match()" .rs .sp -If the \fBpartial_soft\fP (or \fBps\fP) modifier is present on a -\fBpcre2test\fP data line, the PCRE2_PARTIAL_SOFT option is used for the match. -Here is a run of \fBpcre2test\fP that uses the date example quoted above: +The DFA function moves along the subject string character by character, without +backtracking, searching for all possible matches simultaneously. If the end of +the subject is reached before the end of the pattern, there is the possibility +of a partial match. +.P +When PCRE2_PARTIAL_SOFT is set, PCRE2_ERROR_PARTIAL is returned only if there +have been no complete matches. Otherwise, the complete matches are returned. +If PCRE2_PARTIAL_HARD is set, a partial match takes precedence over any +complete matches. The portion of the string that was matched when the longest +partial match was found is set as the first matching string. +.P +Because the DFA function always searches for all possible matches, and there is +no difference between greedy and ungreedy repetition, its behaviour is +different from the \fBpcre2_match()\fP. Consider the string "dog" matched +against this ungreedy pattern: .sp - re> /^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$/ - data> 25jun04\e=ps - 0: 25jun04 - 1: jun - data> 25dec3\e=ps - Partial match: 23dec3 - data> 3ju\e=ps - Partial match: 3ju - data> 3juj\e=ps - No match - data> j\e=ps - No match + /dog(sbody)??/ .sp -The first data string is matched completely, so \fBpcre2test\fP shows the -matched substrings. The remaining four strings do not match the complete -pattern, but the first two are partial matches. Similar output is obtained -if DFA matching is used. -.P -If the \fBpartial_hard\fP (or \fBph\fP) modifier is present on a -\fBpcre2test\fP data line, the PCRE2_PARTIAL_HARD option is set for the match. +Whereas the standard function stops as soon as it finds the complete match for +"dog", the DFA function also finds the partial match for "dogsbody", and so +returns that when PCRE2_PARTIAL_HARD is set. . . .SH "MULTI-SEGMENT MATCHING WITH pcre2_dfa_match()" .rs .sp -When a partial match has been found using a DFA matching function, it is +When a partial match has been found using the DFA matching function, it is possible to continue the match by providing additional subject data and calling the function again with the same compiled regular expression, this time setting the PCRE2_DFA_RESTART option. You must pass the same working space as before, -because this is where details of the previous partial match are stored. Here is -an example using \fBpcre2test\fP: +because this is where details of the previous partial match are stored. You can +set the PCRE2_PARTIAL_SOFT or PCRE2_PARTIAL_HARD options with PCRE2_DFA_RESTART +to continue partial matching over multiple segments. Here is an example using +\fBpcre2test\fP: .sp re> /^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$/ data> 23ja\e=dfa,ps @@ -242,147 +331,10 @@ The first call has "23ja" as the subject, and requests partial matching; the second call has "n05" as the subject for the continued (restarted) match. Notice that when the match is complete, only the last part is shown; PCRE2 does not retain the previously partially-matched string. It is up to the calling -program to do that if it needs to. -.P -That means that, for an unanchored pattern, if a continued match fails, it is -not possible to try again at a new starting point. All this facility is capable -of doing is continuing with the previous match attempt. In the previous -example, if the second set of data is "ug23" the result is no match, even -though there would be a match for "aug23" if the entire string were given at -once. Depending on the application, this may or may not be what you want. -The only way to allow for starting again at the next character is to retain the -matched part of the subject and try a new complete match. -.P -You can set the PCRE2_PARTIAL_SOFT or PCRE2_PARTIAL_HARD options with -PCRE2_DFA_RESTART to continue partial matching over multiple segments. This -facility can be used to pass very long subject strings to the DFA matching -functions. -. -. -.SH "MULTI-SEGMENT MATCHING WITH pcre2_match()" -.rs -.sp -Unlike the DFA function, it is not possible to restart the previous match with -a new segment of data when using \fBpcre2_match()\fP. Instead, new data must be -added to the previous subject string, and the entire match re-run, starting -from the point where the partial match occurred. Earlier data can be discarded. -.P -It is best to use PCRE2_PARTIAL_HARD in this situation, because it does not -treat the end of a segment as the end of the subject when matching \ez, \eZ, -\eb, \eB, and $. Consider an unanchored pattern that matches dates: -.sp - re> /\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed/ - data> The date is 23ja\e=ph - Partial match: 23ja -.sp -At this stage, an application could discard the text preceding "23ja", add on -text from the next segment, and call the matching function again. Unlike the -DFA matching function, the entire matching string must always be available, -and the complete matching process occurs for each call, so more memory and more -processing time is needed. -. -. -.SH "ISSUES WITH MULTI-SEGMENT MATCHING" -.rs -.sp -Certain types of pattern may give problems with multi-segment matching, -whichever matching function is used. -.P -1. If the pattern contains a test for the beginning of a line, you need to pass -the PCRE2_NOTBOL option when the subject string for any call does start at the -beginning of a line. There is also a PCRE2_NOTEOL option, but in practice when -doing multi-segment matching you should be using PCRE2_PARTIAL_HARD, which -includes the effect of PCRE2_NOTEOL. -.P -2. If a pattern contains a lookbehind assertion, characters that precede the -start of the partial match may have been inspected during the matching process. -When using \fBpcre2_match()\fP, sufficient characters must be retained for the -next match attempt. You can ensure that enough characters are retained by doing -the following: -.P -Before doing any matching, find the length of the longest lookbehind in the -pattern by calling \fBpcre2_pattern_info()\fP with the PCRE2_INFO_MAXLOOKBEHIND -option. Note that the resulting count is in characters, not code units. After a -partial match, moving back from the ovector[0] offset in the subject by the -number of characters given for the maximum lookbehind gets you to the earliest -character that must be retained. In a non-UTF or a 32-bit situation, moving -back is just a subtraction, but in UTF-8 or UTF-16 you have to count characters -while moving back through the code units. -.P -Characters before the point you have now reached can be discarded, and after -the next segment has been added to what is retained, you should run the next -match with the \fBstartoffset\fP argument set so that the match begins at the -same point as before. -.P -For example, if the pattern "(?<=123)abc" is partially matched against the -string "xx123ab", the ovector offsets are 5 and 7 ("ab"). The maximum -lookbehind count is 3, so all characters before offset 2 can be discarded. The -value of \fBstartoffset\fP for the next match should be 3. When \fBpcre2test\fP -displays a partial match, it indicates the lookbehind characters with '<' -characters: -.sp - re> "(?<=123)abc" - data> xx123ab\e=ph - Partial match: 123ab - <<< -.P -3. Because a partial match must always contain at least one character, what -might be considered a partial match of an empty string actually gives a "no -match" result. For example: -.sp - re> /c(?<=abc)x/ - data> ab\e=ps - No match -.sp -If the next segment begins "cx", a match should be found, but this will only -happen if characters from the previous segment are retained. For this reason, a -"no match" result should be interpreted as "partial match of an empty string" -when the pattern contains lookbehinds. -.P -4. Matching a subject string that is split into multiple segments may not -always produce exactly the same result as matching over one single long string, -especially when PCRE2_PARTIAL_SOFT is used. The section "Partial Matching and -Word Boundaries" above describes an issue that arises if the pattern ends with -\eb or \eB. Another kind of difference may occur when there are multiple -matching possibilities, because (for PCRE2_PARTIAL_SOFT) a partial match result -is given only when there are no completed matches. This means that as soon as -the shortest match has been found, continuation to a new subject segment is no -longer possible. Consider this \fBpcre2test\fP example: -.sp - re> /dog(sbody)?/ - data> dogsb\e=ps - 0: dog - data> do\e=ps,dfa - Partial match: do - data> gsb\e=ps,dfa,dfa_restart - 0: g - data> dogsbody\e=dfa - 0: dogsbody - 1: dog -.sp -The first data line passes the string "dogsb" to a standard matching function, -setting the PCRE2_PARTIAL_SOFT option. Although the string is a partial match -for "dogsbody", the result is not PCRE2_ERROR_PARTIAL, because the shorter -string "dog" is a complete match. Similarly, when the subject is presented to -a DFA matching function in several parts ("do" and "gsb" being the first two) -the match stops when "dog" has been found, and it is not possible to continue. -On the other hand, if "dogsbody" is presented as a single string, a DFA -matching function finds both matches. -.P -Because of these problems, it is best to use PCRE2_PARTIAL_HARD when matching -multi-segment data. The example above then behaves differently: -.sp - re> /dog(sbody)?/ - data> dogsb\e=ph - Partial match: dogsb - data> do\e=ps,dfa - Partial match: do - data> gsb\e=ph,dfa,dfa_restart - Partial match: gsb -.sp -5. Patterns that contain alternatives at the top level which do not all start -with the same pattern item may not work as expected when PCRE2_DFA_RESTART is -used. For example, consider this pattern: +program to do that if it needs to. This means that, for an unanchored pattern, +if a continued match fails, it is not possible to try again at a new starting +point. All this facility is capable of doing is continuing with the previous +match attempt. For example, consider this pattern: .sp 1234|3789 .sp @@ -391,28 +343,15 @@ alternative is found at offset 3. There is no partial match for the second alternative, because such a match does not start at the same point in the subject string. Attempting to continue with the string "7890" does not yield a match because only those alternatives that match at one point in the subject -are remembered. The problem arises because the start of the second alternative -matches within the first alternative. There is no problem with anchored -patterns or patterns such as: -.sp - 1234|ABCD -.sp -where no string can be a partial match for both alternatives. This is not a -problem if a standard matching function is used, because the entire match has -to be rerun each time: -.sp - re> /1234|3789/ - data> ABC123\e=ph - Partial match: 123 - data> 1237890 - 0: 3789 -.sp -Of course, instead of using PCRE2_DFA_RESTART, the same technique of re-running -the entire match can also be used with the DFA matching function. Another -possibility is to work with two buffers. If a partial match at offset \fIn\fP -in the first buffer is followed by "no match" when PCRE2_DFA_RESTART is used on -the second buffer, you can then try a new match starting at offset \fIn+1\fP in -the first buffer. +are remembered. Depending on the application, this may or may not be what you +want. +.P +If you do want to allow for starting again at the next character, one way of +doing it is to retain some or all of the segment and try a new complete match, +as described for \fBpcre2_match()\fP above. Another possibility is to work with +two buffers. If a partial match at offset \fIn\fP in the first buffer is +followed by "no match" when PCRE2_DFA_RESTART is used on the second buffer, you +can then try a new match starting at offset \fIn+1\fP in the first buffer. . . .SH AUTHOR @@ -429,6 +368,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 22 December 2014 -Copyright (c) 1997-2014 University of Cambridge. +Last updated: 04 September 2019 +Copyright (c) 1997-2019 University of Cambridge. .fi diff --git a/dist2/doc/pcre2pattern.3 b/dist2/doc/pcre2pattern.3 index de8d7cef..dc78e4d1 100644 --- a/dist2/doc/pcre2pattern.3 +++ b/dist2/doc/pcre2pattern.3 @@ -1,4 +1,4 @@ -.TH PCRE2PATTERN 3 "12 February 2019" "PCRE2 10.33" +.TH PCRE2PATTERN 3 "06 October 2020" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH "PCRE2 REGULAR EXPRESSION DETAILS" @@ -52,10 +52,11 @@ single code units, or as multiple UTF-8 or UTF-16 code units. UTF-32 can be specified for the 32-bit library, in which case it constrains the character values to valid Unicode code points. To process UTF strings, PCRE2 must be built to include Unicode support (which is the default). When using UTF strings -you must either call the compiling function with the PCRE2_UTF option, or the -pattern must start with the special sequence (*UTF), which is equivalent to -setting the relevant option. How setting a UTF mode affects pattern matching is -mentioned in several places below. There is also a summary of features in the +you must either call the compiling function with one or both of the PCRE2_UTF +or PCRE2_MATCH_INVALID_UTF options, or the pattern must start with the special +sequence (*UTF), which is equivalent to setting the relevant PCRE2_UTF. How +setting a UTF mode affects pattern matching is mentioned in several places +below. There is also a summary of features in the .\" HREF \fBpcre2unicode\fP .\" @@ -74,7 +75,8 @@ Another special sequence that may appear at the start of a pattern is (*UCP). This has the same effect as setting the PCRE2_UCP option: it causes sequences such as \ed and \ew to use Unicode properties to determine character types, instead of recognizing only characters with codes less than 256 via a lookup -table. +table. If also causes upper/lower casing operations to use Unicode properties +for characters with code points greater than 127, even when UTF is not set. .P Some applications that allow their users to supply patterns may wish to restrict them for security reasons. If the PCRE2_NEVER_UCP option is passed to @@ -261,8 +263,11 @@ corresponding characters in the subject. As a trivial example, the pattern The quick brown fox .sp matches a portion of a subject string that is identical to itself. When -caseless matching is specified (the PCRE2_CASELESS option), letters are matched -independently of case. +caseless matching is specified (the PCRE2_CASELESS option or (?i) within the +pattern), letters are matched independently of case. Note that there are two +ASCII characters, K and S, that, in addition to their lower case ASCII +equivalents, are case-equivalent with Unicode U+212A (Kelvin sign) and U+017F +(long S) respectively when either PCRE2_UTF or PCRE2_UCP is set. .P The power of regular expressions comes from the ability to include wild cards, character classes, alternatives, and repetitions in the pattern. These are @@ -296,6 +301,22 @@ a character class the only metacharacters are: [ POSIX character class (if followed by POSIX syntax) ] terminates the character class .sp +If a pattern is compiled with the PCRE2_EXTENDED option, most white space in +the pattern, other than in a character class, and characters between a # +outside a character class and the next newline, inclusive, are ignored. An +escaping backslash can be used to include a white space or a # character as +part of the pattern. If the PCRE2_EXTENDED_MORE option is set, the same +applies, but in addition unescaped space and horizontal tab characters are +ignored inside a character class. Note: only these two characters are ignored, +not the full set of pattern white space characters that are ignored outside a +character class. Option settings can be changed within a pattern; see the +section entitled +.\" HTML <a href="#internaloptions"> +.\" </a> +"Internal Option Setting" +.\" +below. +.P The following sections describe the use of each of the metacharacters. . . @@ -313,15 +334,9 @@ would otherwise be interpreted as a metacharacter, so it is always safe to precede a non-alphanumeric with backslash to specify that it stands for itself. In particular, if you want to match a backslash, you write \e\e. .P -In a UTF mode, only ASCII digits and letters have any special meaning after a -backslash. All other characters (in particular, those whose code points are -greater than 127) are treated as literals. -.P -If a pattern is compiled with the PCRE2_EXTENDED option, most white space in -the pattern (other than in a character class), and characters between a # -outside a character class and the next newline, inclusive, are ignored. An -escaping backslash can be used to include a white space or # character as part -of the pattern. +Only ASCII digits and letters have any special meaning after a backslash. All +other characters (in particular, those whose code points are greater than 127) +are treated as literals. .P If you want to treat all characters in a sequence as literals, you can do so by putting them between \eQ and \eE. This is different from Perl in that $ and @ @@ -398,11 +413,11 @@ PCRE2_EXTRA_ALT_BSUX has the same effect as PCRE2_ALT_BSUX and, in addition, There may be any number of hexadecimal digits. This syntax is from ECMAScript 6. .P -The \eN{U+hhh..} escape sequence is recognized only when the PCRE2_UTF option -is set, that is, when PCRE2 is operating in a Unicode mode. Perl also uses -\eN{name} to specify characters by Unicode name; PCRE2 does not support this. -Note that when \eN is not followed by an opening brace (curly bracket) it has -an entirely different meaning, matching any character that is not a newline. +The \eN{U+hhh..} escape sequence is recognized only when PCRE2 is operating in +UTF mode. Perl also uses \eN{name} to specify characters by Unicode name; PCRE2 +does not support this. Note that when \eN is not followed by an opening brace +(curly bracket) it has an entirely different meaning, matching any character +that is not a newline. .P There are some legacy applications where the escape sequence \er is expected to match a newline. If the PCRE2_EXTRA_ESCAPED_CR_IS_LF option is set, \er in a @@ -812,6 +827,7 @@ Caucasian_Albanian, Chakma, Cham, Cherokee, +Chorasmian, Common, Coptic, Cuneiform, @@ -819,10 +835,12 @@ Cypriot, Cyrillic, Deseret, Devanagari, +Dives_Akuru, Dogra, Duployan, Egyptian_Hieroglyphs, Elbasan, +Elymaic, Ethiopic, Georgian, Glagolitic, @@ -849,6 +867,7 @@ Kannada, Katakana, Kayah_Li, Kharoshthi, +Khitan_Small_Script, Khmer, Khojki, Khudawadi, @@ -880,10 +899,12 @@ Mro, Multani, Myanmar, Nabataean, +Nandinagari, New_Tai_Lue, Newa, Nko, Nushu, +Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungarian, @@ -935,7 +956,9 @@ Tirhuta, Ugaritic, Unknown, Vai, +Wancho, Warang_Citi, +Yezidi, Yi, Zanabazar_Square. .P @@ -1145,8 +1168,9 @@ For example, when the pattern .sp matches "foobar", the first substring is still set to "foo". .P -Perl documents that the use of \eK within assertions is "not well defined". In -PCRE2, \eK is acted upon when it occurs inside positive assertions, but is +Perl used to document that the use of \eK within lookaround assertions is "not +well defined", but from version 5.32.0 Perl does not support this usage at all. +In PCRE2, \eK is acted upon when it occurs inside positive assertions, but is ignored in negative assertions. Note that when a pattern such as (?=ab\eK) matches, the reported start of the match can be greater than the end of the match. Using \eK in a lookbehind assertion at the start of a pattern can also @@ -1352,7 +1376,7 @@ with \eC in UTF-8 or UTF-16 mode means that the rest of the string may start with a malformed UTF character. This has undefined results, because PCRE2 assumes that it is matching character by character in a valid UTF string (by default it checks the subject string's validity at the start of processing -unless the PCRE2_NO_UTF_CHECK option is used). +unless the PCRE2_NO_UTF_CHECK or PCRE2_MATCH_INVALID_UTF option is used). .P An application can lock out the use of \eC by setting the PCRE2_NEVER_BACKSLASH_C option when compiling a pattern. It is also possible to @@ -1426,7 +1450,10 @@ Characters in a class may be specified by their code points using \eo, \ex, or \eN{U+hh..} in the usual way. When caseless matching is set, any letters in a class represent both their upper case and lower case versions, so for example, a caseless [aeiou] matches "A" as well as "a", and a caseless [^aeiou] does not -match "A", whereas a caseful version would. +match "A", whereas a caseful version would. Note that there are two ASCII +characters, K and S, that, in addition to their lower case ASCII equivalents, +are case-equivalent with Unicode U+212A (Kelvin sign) and U+017F (long S) +respectively when either PCRE2_UTF or PCRE2_UCP is set. .P Characters that might indicate line breaks are never treated in any special way when matching character classes, whatever line-ending sequence is in use, and @@ -1638,6 +1665,7 @@ that succeeds is used. If the alternatives are within a group alternative in the group. . . +.\" HTML <a name="internaloptions"></a> .SH "INTERNAL OPTION SETTING" .rs .sp @@ -1896,12 +1924,21 @@ are permitted for groups with the same number, for example: (?|(?<AA>aa)|(?<AA>bb)) .sp The duplicate name constraint can be disabled by setting the PCRE2_DUPNAMES -option at compile time, or by the use of (?J) within the pattern. Duplicate -names can be useful for patterns where only one instance of the named capture -group can match. Suppose you want to match the name of a weekday, either as a -3-letter abbreviation or as the full name, and in both cases you want to -extract the abbreviation. This pattern (ignoring the line breaks) does the job: +option at compile time, or by the use of (?J) within the pattern, as described +in the section entitled +.\" HTML <a href="#internaloptions"> +.\" </a> +"Internal Option Setting" +.\" +above. +.P +Duplicate names can be useful for patterns where only one instance of the named +capture group can match. Suppose you want to match the name of a weekday, +either as a 3-letter abbreviation or as the full name, and in both cases you +want to extract the abbreviation. This pattern (ignoring the line breaks) does +the job: .sp + (?J) (?<DN>Mon|Fri|Sun)(?:day)?| (?<DN>Tue)(?:sday)?| (?<DN>Wed)(?:nesday)?| @@ -1921,7 +1958,7 @@ they appear in the overall pattern. The first one that is set is used for the reference. For example, this pattern matches both "foofoo" and "barbar" but not "foobar" or "barfoo": .sp - (?:(?<n>foo)|(?<n>bar))\ek<n> + (?J)(?:(?<n>foo)|(?<n>bar))\ek<n> .sp .P If you make a subroutine call to a non-unique named group, the one that @@ -1960,7 +1997,7 @@ items: an escape such as \ed or \epL that matches a single character a character class a backreference - a parenthesized group (including most assertions) + a parenthesized group (including lookaround assertions) a subroutine call (recursive or otherwise) .sp The general repetition quantifier specifies a minimum and maximum number of @@ -2021,8 +2058,10 @@ no characters with a quantifier that has no upper limit, for example: .sp Earlier versions of Perl and PCRE1 used to give an error at compile time for such patterns. However, because there are cases where this can be useful, such -patterns are now accepted, but if any repetition of the group does in fact -match no characters, the loop is forcibly broken. +patterns are now accepted, but whenever an iteration of such a group matches no +characters, matching moves on to the next item in the pattern instead of +repeatedly matching an empty string. This does not prevent backtracking into +any of the iterations if a subsequent item fails to match. .P By default, quantifiers are "greedy", that is, they match as much as possible (up to the maximum number of permitted times), without causing the rest of the @@ -2339,14 +2378,14 @@ the first iteration does not need to match the backreference. This can be done using alternation, as in the example above, or by a quantifier with a minimum of zero. .P -Backreferences of this type cause the group that they reference to be treated -as an +For versions of PCRE2 less than 10.25, backreferences of this type used to +cause the group that they reference to be treated as an .\" HTML <a href="#atomicgroup"> .\" </a> atomic group. .\" -Once the whole group has been matched, a subsequent matching failure cannot -cause backtracking into the middle of the group. +This restriction no longer applies, and backtracking into such groups can occur +as normal. . . .\" HTML <a name="bigassertions"></a> @@ -2367,9 +2406,19 @@ those that look behind it, and in each case an assertion may be positive (must match for the assertion to be true) or negative (must not match for the assertion to be true). An assertion group is matched in the normal way, and if it is true, matching continues after it, but with the matching position -in the subject string is was it was before the assertion was processed. +in the subject string reset to what it was before the assertion was processed. +.P +The Perl-compatible lookaround assertions are atomic. If an assertion is true, +but there is a subsequent matching failure, there is no backtracking into the +assertion. However, there are some cases where non-atomic assertions can be +useful. PCRE2 has some support for these, described in the section entitled +.\" HTML <a href="#nonatomicassertions"> +.\" </a> +"Non-atomic assertions" +.\" +below, but they are not Perl-compatible. .P -A lookaround assertion may also appear as the condition in a +A lookaround assertion may appear as the condition in a .\" HTML <a href="#conditions"> .\" </a> conditional group @@ -2404,36 +2453,23 @@ the "no" branch of the condition. For other failing negative assertions, control passes to the previous backtracking point, thus discarding any captured strings within the assertion. .P -For compatibility with Perl, most assertion groups may be repeated; though it -makes no sense to assert the same thing several times, the side effect of -capturing may occasionally be useful. However, an assertion that forms the -condition for a conditional group may not be quantified. In practice, for -other assertions, there only three cases: -.sp -(1) If the quantifier is {0}, the assertion is never obeyed during matching. -However, it may contain internal capture groups that are called from elsewhere -via the -.\" HTML <a href="#groupsassubroutines"> -.\" </a> -subroutine mechanism. -.\" -.sp -(2) If quantifier is {0,n} where n is greater than zero, it is treated as if it -were {0,1}. At run time, the rest of the pattern match is tried with and -without the assertion, the order depending on the greediness of the quantifier. -.sp -(3) If the minimum repetition is greater than zero, the quantifier is ignored. -The assertion is obeyed just once when encountered during matching. +Most assertion groups may be repeated; though it makes no sense to assert the +same thing several times, the side effect of capturing in positive assertions +may occasionally be useful. However, an assertion that forms the condition for +a conditional group may not be quantified. PCRE2 used to restrict the +repetition of assertions, but from release 10.35 the only restriction is that +an unlimited maximum repetition is changed to be one more than the minimum. For +example, {3,} is treated as {3,4}. . . .SS "Alphabetic assertion names" .rs .sp -Traditionally, symbolic sequences such as (?= and (?<= have been used to specify -lookaround assertions. Perl 5.28 introduced some experimental alphabetic -alternatives which might be easier to remember. They all start with (* instead -of (? and must be written using lower case letters. PCRE2 supports the -following synonyms: +Traditionally, symbolic sequences such as (?= and (?<= have been used to +specify lookaround assertions. Perl 5.28 introduced some experimental +alphabetic alternatives which might be easier to remember. They all start with +(* instead of (? and must be written using lower case letters. PCRE2 supports +the following synonyms: .sp (*positive_lookahead: or (*pla: is the same as (?= (*negative_lookahead: or (*nla: is the same as (?! @@ -2610,6 +2646,68 @@ is another pattern that matches "foo" preceded by three digits and any three characters that are not "999". . . +.\" HTML <a name="nonatomicassertions"></a> +.SH "NON-ATOMIC ASSERTIONS" +.rs +.sp +The traditional Perl-compatible lookaround assertions are atomic. That is, if +an assertion is true, but there is a subsequent matching failure, there is no +backtracking into the assertion. However, there are some cases where non-atomic +positive assertions can be useful. PCRE2 provides these using the following +syntax: +.sp + (*non_atomic_positive_lookahead: or (*napla: or (?* + (*non_atomic_positive_lookbehind: or (*naplb: or (?<* +.sp +Consider the problem of finding the right-most word in a string that also +appears earlier in the string, that is, it must appear at least twice in total. +This pattern returns the required result as captured substring 1: +.sp + ^(?x)(*napla: .* \eb(\ew++)) (?> .*? \eb\e1\eb ){2} +.sp +For a subject such as "word1 word2 word3 word2 word3 word4" the result is +"word3". How does it work? At the start, ^(?x) anchors the pattern and sets the +"x" option, which causes white space (introduced for readability) to be +ignored. Inside the assertion, the greedy .* at first consumes the entire +string, but then has to backtrack until the rest of the assertion can match a +word, which is captured by group 1. In other words, when the assertion first +succeeds, it captures the right-most word in the string. +.P +The current matching point is then reset to the start of the subject, and the +rest of the pattern match checks for two occurrences of the captured word, +using an ungreedy .*? to scan from the left. If this succeeds, we are done, but +if the last word in the string does not occur twice, this part of the pattern +fails. If a traditional atomic lookhead (?= or (*pla: had been used, the +assertion could not be re-entered, and the whole match would fail. The pattern +would succeed only if the very last word in the subject was found twice. +.P +Using a non-atomic lookahead, however, means that when the last word does not +occur twice in the string, the lookahead can backtrack and find the second-last +word, and so on, until either the match succeeds, or all words have been +tested. +.P +Two conditions must be met for a non-atomic assertion to be useful: the +contents of one or more capturing groups must change after a backtrack into the +assertion, and there must be a backreference to a changed group later in the +pattern. If this is not the case, the rest of the pattern match fails exactly +as before because nothing has changed, so using a non-atomic assertion just +wastes resources. +.P +There is one exception to backtracking into a non-atomic assertion. If an +(*ACCEPT) control verb is triggered, the assertion succeeds atomically. That +is, a subsequent match failure cannot backtrack into the assertion. +.P +Non-atomic assertions are not supported by the alternative matching function +\fBpcre2_dfa_match()\fP. They are supported by JIT, but only if they do not +contain any control verbs such as (*ACCEPT). (This may change in future). Note +that assertions that appear as conditions for +.\" HTML <a href="#conditions"> +.\" </a> +conditional groups +.\" +(see below) must be atomic. +. +. .SH "SCRIPT RUNS" .rs .sp @@ -2861,8 +2959,15 @@ than two digits. .sp If the condition is not in any of the above formats, it must be a parenthesized assertion. This may be a positive or negative lookahead or lookbehind -assertion. Consider this pattern, again containing non-significant white space, -and with the two alternatives on the second line: +assertion. However, it must be a traditional atomic assertion, not one of the +PCRE2-specific +.\" HTML <a href="#nonatomicassertions"> +.\" </a> +non-atomic assertions. +.\" +.P +Consider this pattern, again containing non-significant white space, and with +the two alternatives on the second line: .sp (?(?=[^a-z]*[a-z]) \ed{2}-[a-z]{3}-\ed{2} | \ed{2}-\ed{2}-\ed{2} ) @@ -3261,8 +3366,8 @@ The doubling is removed before the string is passed to the callout function. There are a number of special "Backtracking Control Verbs" (to use Perl's terminology) that modify the behaviour of backtracking during matching. They are generally of the form (*VERB) or (*VERB:NAME). Some verbs take either form, -possibly behaving differently depending on whether or not a name is present. -The names are not required to be unique within the pattern. +and may behave differently depending on whether or not a name argument is +present. The names are not required to be unique within the pattern. .P By default, for compatibility with Perl, a name is any sequence of characters that does not include a closing parenthesis. The name is not processed in @@ -3286,7 +3391,8 @@ PCRE2_ALT_VERBNAMES is also set. The maximum length of a name is 255 in the 8-bit library and 65535 in the 16-bit and 32-bit libraries. If the name is empty, that is, if the closing parenthesis immediately follows the colon, the effect is as if the colon were -not there. Any number of these verbs may occur in a pattern. +not there. Any number of these verbs may occur in a pattern. Except for +(*ACCEPT), they may not be quantified. .P Since these verbs are specifically related to backtracking, most of them can be used only when the pattern is to be matched using the traditional matching @@ -3360,6 +3466,18 @@ example: This matches "AB", "AAD", or "ACD"; when it matches "AB", "B" is captured by the outer parentheses. .P +(*ACCEPT) is the only backtracking verb that is allowed to be quantified +because an ungreedy quantification with a minimum of zero acts only when a +backtrack happens. Consider, for example, +.sp + (A(*ACCEPT)??B)C +.sp +where A, B, and C may be complex expressions. After matching "A", the matcher +processes "BC"; if that fails, causing a backtrack, (*ACCEPT) is triggered and +the match succeeds. In both cases, all but C is captured. Whereas (*COMMIT) +(see below) means "fail on backtrack", a repeated (*ACCEPT) of this type means +"succeed on backtrack". +.P \fBWarning:\fP (*ACCEPT) should not be used within a script run group, because it causes an immediate exit from the group, bypassing the script run checking. .sp @@ -3376,8 +3494,9 @@ nearest equivalent is the callout feature, as for example in this pattern: A match with the string "aaaa" always fails, but the callout is taken before each backtrack happens (in this example, 10 times). .P -(*ACCEPT:NAME) and (*FAIL:NAME) are treated as (*MARK:NAME)(*ACCEPT) and -(*MARK:NAME)(*FAIL), respectively. +(*ACCEPT:NAME) and (*FAIL:NAME) behave the same as (*MARK:NAME)(*ACCEPT) and +(*MARK:NAME)(*FAIL), respectively, that is, a (*MARK) is recorded just before +the verb acts. . . .SS "Recording which path was taken" @@ -3543,6 +3662,11 @@ next attempt at "c". Note that a possessive quantifer does not have the same effect as this example; although it would suppress backtracking during the first match attempt, the second attempt would start at the second character instead of skipping on to "c". +.P +If (*SKIP) is used to specify a new starting position that is the same as the +starting position of the current match, or (by being inside a lookbehind) +earlier, the position specified by (*SKIP) is ignored, and instead the normal +"bumpalong" occurs. .sp (*SKIP:NAME) .sp @@ -3700,11 +3824,22 @@ a positive assertion and false for a negative one; captured substrings are retained in both cases. .P The remaining verbs act only when a later failure causes a backtrack to -reach them. This means that their effect is confined to the assertion, -because lookaround assertions are atomic. A backtrack that occurs after an -assertion is complete does not jump back into the assertion. Note in particular -that a (*MARK) name that is set in an assertion is not "seen" by an instance of -(*SKIP:NAME) latter in the pattern. +reach them. This means that, for the Perl-compatible assertions, their effect +is confined to the assertion, because Perl lookaround assertions are atomic. A +backtrack that occurs after such an assertion is complete does not jump back +into the assertion. Note in particular that a (*MARK) name that is set in an +assertion is not "seen" by an instance of (*SKIP:NAME) later in the pattern. +.P +PCRE2 now supports non-atomic positive assertions, as described in the section +entitled +.\" HTML <a href="#nonatomicassertions"> +.\" </a> +"Non-atomic assertions" +.\" +above. These assertions must be standalone (not used as conditions). They are +not Perl-compatible. For these assertions, a later backtrack does jump back +into the assertion, and therefore verbs such as (*COMMIT) can be triggered by +backtracks from later in the pattern. .P The effect of (*THEN) is not allowed to escape beyond an assertion. If there are no more branches to try, (*THEN) causes a positive assertion to be false, @@ -3763,6 +3898,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 12 February 2019 -Copyright (c) 1997-2019 University of Cambridge. +Last updated: 06 October 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/doc/pcre2syntax.3 b/dist2/doc/pcre2syntax.3 index 70538e4e..70764620 100644 --- a/dist2/doc/pcre2syntax.3 +++ b/dist2/doc/pcre2syntax.3 @@ -1,4 +1,4 @@ -.TH PCRE2SYNTAX 3 "11 February 2019" "PCRE2 10.33" +.TH PCRE2SYNTAX 3 "28 December 2019" "PCRE2 10.35" .SH NAME PCRE2 - Perl-compatible regular expressions (revised API) .SH "PCRE2 REGULAR EXPRESSION SYNTAX SUMMARY" @@ -193,6 +193,7 @@ Caucasian_Albanian, Chakma, Cham, Cherokee, +Chorasmian, Common, Coptic, Cuneiform, @@ -200,10 +201,12 @@ Cypriot, Cyrillic, Deseret, Devanagari, +Dives_Akuru, Dogra, Duployan, Egyptian_Hieroglyphs, Elbasan, +Elymaic, Ethiopic, Georgian, Glagolitic, @@ -230,6 +233,7 @@ Kannada, Katakana, Kayah_Li, Kharoshthi, +Khitan_Small_Script, Khmer, Khojki, Khudawadi, @@ -261,10 +265,12 @@ Mro, Multani, Myanmar, Nabataean, +Nandinagari, New_Tai_Lue, Newa, Nko, Nushu, +Nyakeng_Puachue_Hmong, Ogham, Ol_Chiki, Old_Hungarian, @@ -315,7 +321,9 @@ Tifinagh, Tirhuta, Ugaritic, Vai, +Wancho, Warang_Citi, +Yezidi, Yi, Zanabazar_Square. . @@ -437,7 +445,7 @@ Changes of these options within a group are automatically cancelled at the end of the group. .sp (?i) caseless - (?J) allow duplicate names + (?J) allow duplicate named groups (?m) multiline (?n) no auto capture (?s) single line (dotall) @@ -522,6 +530,20 @@ setting with a similar syntax. Each top-level branch of a lookbehind must be of a fixed length. . . +.SH "NON-ATOMIC LOOKAROUND ASSERTIONS" +.rs +.sp +These assertions are specific to PCRE2 and are not Perl-compatible. +.sp + (?*...) ) + (*napla:...) ) synonyms + (*non_atomic_positive_lookahead:...) ) +.sp + (?<*...) ) + (*naplb:...) ) synonyms + (*non_atomic_positive_lookbehind:...) ) +. +. .SH "SCRIPT RUNS" .rs .sp @@ -654,6 +676,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 11 February 2019 +Last updated: 28 December 2019 Copyright (c) 1997-2019 University of Cambridge. .fi diff --git a/dist2/doc/pcre2test.1 b/dist2/doc/pcre2test.1 index 568e7054..c63f03df 100644 --- a/dist2/doc/pcre2test.1 +++ b/dist2/doc/pcre2test.1 @@ -1,4 +1,4 @@ -.TH PCRE2TEST 1 "11 March 2019" "PCRE 10.33" +.TH PCRE2TEST 1 "14 September 2020" "PCRE 10.36" .SH NAME pcre2test - a program for testing Perl-compatible regular expressions. .SH SYNOPSIS @@ -202,17 +202,23 @@ compiled pattern is given after compilation. Behave as if each pattern line has the \fBjit\fP modifier; after successful compilation, each pattern is passed to the just-in-time compiler, if available. .TP 10 +\fB-jitfast\fP +Behave as if each pattern line has the \fBjitfast\fP modifier; after +successful compilation, each pattern is passed to the just-in-time compiler, if +available, and each subject line is passed directly to the JIT matcher via its +"fast path". +.TP 10 \fB-jitverify\fP Behave as if each pattern line has the \fBjitverify\fP modifier; after successful compilation, each pattern is passed to the just-in-time compiler, if -available, and the use of JIT is verified. +available, and the use of JIT for matching is verified. .TP 10 \fB-LM\fP List modifiers: write a list of available pattern and subject modifiers to the standard output, then exit with zero exit code. All other options are ignored. If both -C and -LM are present, whichever is first is recognized. .TP 10 -\fB-pattern\fB \fImodifier-list\fP +\fB-pattern\fP \fImodifier-list\fP Behave as if each pattern line contains the given modifiers. .TP 10 \fB-q\fP @@ -267,7 +273,7 @@ test data, command lines that begin with # may appear. This file format, with some restrictions, can also be processed by the \fBperltest.sh\fP script that is distributed with PCRE2 as a means of checking that the behaviour of PCRE2 and Perl is the same. For a specification of \fBperltest.sh\fP, see the -comments near its beginning. +comments near its beginning. See also the #perltest command below. .P When the input is a terminal, \fBpcre2test\fP prompts for each line of input, using "re>" to prompt for regular expression patterns, and "data>" to prompt @@ -321,6 +327,12 @@ described in the section entitled "Saving and restoring compiled patterns" below. .\" .sp + #loadtables <filename> +.sp +This command is used to load a set of binary character tables that can be +accessed by the tables=3 qualifier. Such tables can be created by the +\fBpcre2_dftables\fP program with the -b option. +.sp #newline_default [<newline-list>] .sp When PCRE2 is built, a default newline convention can be specified. This @@ -357,14 +369,19 @@ patterns. Modifiers on a pattern can change these settings. .sp #perltest .sp -The appearance of this line causes all subsequent modifier settings to be -checked for compatibility with the \fBperltest.sh\fP script, which is used to -confirm that Perl gives the same results as PCRE2. Also, apart from comment -lines, #pattern commands, and #subject commands that set or unset "mark", no -command lines are permitted, because they and many of the modifiers are -specific to \fBpcre2test\fP, and should not be used in test files that are also -processed by \fBperltest.sh\fP. The \fB#perltest\fP command helps detect tests -that are accidentally put in the wrong file. +This line is used in test files that can also be processed by \fBperltest.sh\fP +to confirm that Perl gives the same results as PCRE2. Subsequent tests are +checked for the use of \fBpcre2test\fP features that are incompatible with the +\fBperltest.sh\fP script. +.P +Patterns must use '/' as their delimiter, and only certain modifiers are +supported. Comment lines, #pattern commands, and #subject commands that set or +unset "mark" are recognized and acted on. The #perltest, #forbid_utf, and +#newline_default commands, which are needed in the relevant pcre2test files, +are silently ignored. All other command lines are ignored, but give a warning +message. The \fB#perltest\fP command helps detect tests that are accidentally +put in the wrong file or use the wrong delimiter. For more details of the +\fBperltest.sh\fP script see the comments it contains. .sp #pop [<modifiers>] #popcopy [<modifiers>] @@ -572,6 +589,7 @@ for a description of the effects of these options. firstline set PCRE2_FIRSTLINE literal set PCRE2_LITERAL match_line set PCRE2_EXTRA_MATCH_LINE + match_invalid_utf set PCRE2_MATCH_INVALID_UTF match_unset_backref set PCRE2_MATCH_UNSET_BACKREF match_word set PCRE2_EXTRA_MATCH_WORD /m multiline set PCRE2_MULTILINE @@ -631,7 +649,7 @@ heavily used in the test files. pushcopy push a copy onto the stack stackguard=<number> test the stackguard feature subject_literal treat all subject lines as literal - tables=[0|1|2] select internal tables + tables=[0|1|2|3] select internal tables use_length do not zero-terminate the pattern utf8_input treat input as UTF-8 .sp @@ -694,7 +712,9 @@ options, the line is omitted. "First code unit" is where any match must start; if there is more than one they are listed as "starting code units". "Last code unit" is the last literal code unit that must be present in any match. This is not necessarily the last character. These lines are omitted if no starting or -ending code units are recorded. +ending code units are recorded. The subject length line is omitted when +\fBno_start_optimize\fP is set because the minimum length is not calculated +when it can never be used. .P The \fBframesize\fP modifier shows the size, in bytes, of the storage frames used by \fBpcre2_match()\fP for handling backtracking. The size depends on the @@ -979,18 +999,20 @@ be aborted. .rs .sp The value specified for the \fBtables\fP modifier must be one of the digits 0, -1, or 2. It causes a specific set of built-in character tables to be passed to -\fBpcre2_compile()\fP. This is used in the PCRE2 tests to check behaviour with -different character tables. The digit specifies the tables as follows: +1, 2, or 3. It causes a specific set of built-in character tables to be passed +to \fBpcre2_compile()\fP. This is used in the PCRE2 tests to check behaviour +with different character tables. The digit specifies the tables as follows: .sp 0 do not pass any special character tables 1 the default ASCII tables, as distributed in pcre2_chartables.c.dist 2 a set of tables defining ISO 8859 characters + 3 a set of tables loaded by the #loadtables command .sp -In table 2, some characters whose codes are greater than 128 are identified as -letters, digits, spaces, etc. Setting alternate character tables and a locale -are mutually exclusive. +In tables 2, some characters whose codes are greater than 128 are identified as +letters, digits, spaces, etc. Tables 3 can be used only after a +\fB#loadtables\fP command has loaded them from a binary file. Setting alternate +character tables and a locale are mutually exclusive. . . .SS "Setting certain match controls" @@ -1002,24 +1024,27 @@ modifier list, in which case they are applied to every subject line that is processed with that pattern. These modifiers do not affect the compilation process. .sp - aftertext show text after match - allaftertext show text after captures - allcaptures show all captures - allvector show the entire ovector - allusedtext show all consulted text - altglobal alternative global matching - /g global global matching - jitstack=<n> set size of JIT stack - mark show mark values - replace=<string> specify a replacement string - startchar show starting character when relevant - substitute_callout use substitution callouts - substitute_extended use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n - substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH - substitute_stop=<n> skip substitution number n and greater - substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET - substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY + aftertext show text after match + allaftertext show text after captures + allcaptures show all captures + allvector show the entire ovector + allusedtext show all consulted text + altglobal alternative global matching + /g global global matching + jitstack=<n> set size of JIT stack + mark show mark values + replace=<string> specify a replacement string + startchar show starting character when relevant + substitute_callout use substitution callouts + substitute_extended use PCRE2_SUBSTITUTE_EXTENDED + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED + substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution <n> + substitute_stop=<n> skip substitution <n> and following + substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET + substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY .sp These modifiers may not appear in a \fB#pattern\fP command. If you want them as defaults, set them in a \fB#subject\fP command. @@ -1192,8 +1217,11 @@ pattern. startoffset=<n> same as offset=<n> substitute_callout use substitution callouts substitute_extedded use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution number n substitute_stop=<n> skip substitution number n and greater substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY @@ -1217,22 +1245,27 @@ well as the main matched substring. In each case the remainder is output on the following line with a plus character following the capture number. .P The \fBallusedtext\fP modifier requests that all the text that was consulted -during a successful pattern match by the interpreter should be shown. This -feature is not supported for JIT matching, and if requested with JIT it is -ignored (with a warning message). Setting this modifier affects the output if -there is a lookbehind at the start of a match, or a lookahead at the end, or if -\eK is used in the pattern. Characters that precede or follow the start and end -of the actual match are indicated in the output by '<' or '>' characters -underneath them. Here is an example: +during a successful pattern match by the interpreter should be shown, for both +full and partial matches. This feature is not supported for JIT matching, and +if requested with JIT it is ignored (with a warning message). Setting this +modifier affects the output if there is a lookbehind at the start of a match, +or, for a complete match, a lookahead at the end, or if \eK is used in the +pattern. Characters that precede or follow the start and end of the actual +match are indicated in the output by '<' or '>' characters underneath them. +Here is an example: .sp re> /(?<=pqr)abc(?=xyz)/ data> 123pqrabcxyz456\e=allusedtext 0: pqrabcxyz <<< >>> + data> 123pqrabcxy\e=ph,allusedtext + Partial match: pqrabcxy + <<< .sp -This shows that the matched string is "abc", with the preceding and following -strings "pqr" and "xyz" having been consulted during the match (when processing -the assertions). +The first, complete match shows that the matched string is "abc", with the +preceding and following strings "pqr" and "xyz" having been consulted during +the match (when processing the assertions). The partial match can indicate only +the preceding string. .P The \fBstartchar\fP modifier requests that the starting character for the match be indicated, if it is different to the start of the matched string. The only @@ -1351,9 +1384,10 @@ by name. .rs .sp If the \fBreplace\fP modifier is set, the \fBpcre2_substitute()\fP function is -called instead of one of the matching functions. Note that replacement strings -cannot contain commas, because a comma signifies the end of a modifier. This is -not thought to be an issue in a test program. +called instead of one of the matching functions (or after one call of +\fBpcre2_match()\fP in the case of PCRE2_SUBSTITUTE_MATCHED). Note that +replacement strings cannot contain commas, because a comma signifies the end of +a modifier. This is not thought to be an issue in a test program. .P Unlike subject strings, \fBpcre2test\fP does not process replacement strings for escape sequences. In UTF mode, a replacement string is checked to see if it @@ -1367,10 +1401,18 @@ for \fBpcre2_substitute()\fP: .sp global PCRE2_SUBSTITUTE_GLOBAL substitute_extended PCRE2_SUBSTITUTE_EXTENDED + substitute_literal PCRE2_SUBSTITUTE_LITERAL + substitute_matched PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only PCRE2_SUBSTITUTE_REPLACEMENT_ONLY substitute_unknown_unset PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty PCRE2_SUBSTITUTE_UNSET_EMPTY .sp +See the +.\" HREF +\fBpcre2api\fP +.\" +documentation for details of these options. .P After a successful substitution, the modified string is output, preceded by the number of replacements. This may be zero if there were no matches. Here is a @@ -1745,7 +1787,7 @@ restart the match with additional subject data by means of the \fBdfa_restart\fP modifier. For example: .sp re> /^\ed?\ed(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\ed\ed$/ - data> 23ja\e=P,dfa + data> 23ja\e=ps,dfa Partial match: 23ja data> n05\e=dfa,dfa_restart 0: n05 @@ -2059,6 +2101,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 11 March 2019 -Copyright (c) 1997-2019 University of Cambridge. +Last updated: 14 September 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/doc/pcre2test.txt b/dist2/doc/pcre2test.txt index cbe3528a..645156e7 100644 --- a/dist2/doc/pcre2test.txt +++ b/dist2/doc/pcre2test.txt @@ -13,8 +13,8 @@ SYNOPSIS but it can also be used for experimenting with regular expressions. This document describes the features of the test program; for details of the regular expressions themselves, see the pcre2pattern documenta- - tion. For details of the PCRE2 library function calls and their - options, see the pcre2api documentation. + tion. For details of the PCRE2 library function calls and their op- + tions, see the pcre2api documentation. The input for pcre2test is a sequence of regular expression patterns and subject strings to be matched. There are also command lines for @@ -33,26 +33,26 @@ SYNOPSIS which are specifically designed for use in conjunction with the test script and data files that are distributed as part of PCRE2. All the modifiers are documented here, some without much justification, but - many of them are unlikely to be of use except when testing the - libraries. + many of them are unlikely to be of use except when testing the li- + braries. PCRE2's 8-BIT, 16-BIT AND 32-BIT LIBRARIES Different versions of the PCRE2 library can be built to support charac- ter strings that are encoded in 8-bit, 16-bit, or 32-bit code units. - One, two, or all three of these libraries may be simultaneously - installed. The pcre2test program can be used to test all the libraries. + One, two, or all three of these libraries may be simultaneously in- + stalled. The pcre2test program can be used to test all the libraries. However, its own input and output are always in 8-bit format. When testing the 16-bit or 32-bit libraries, patterns and subject strings - are converted to 16-bit or 32-bit format before being passed to the - library functions. Results are converted back to 8-bit code units for + are converted to 16-bit or 32-bit format before being passed to the li- + brary functions. Results are converted back to 8-bit code units for output. In the rest of this document, the names of library functions and struc- - tures are given in generic form, for example, pcre_compile(). The - actual names used in the libraries have a suffix _8, _16, or _32, as - appropriate. + tures are given in generic form, for example, pcre_compile(). The ac- + tual names used in the libraries have a suffix _8, _16, or _32, as ap- + propriate. INPUT ENCODING @@ -70,18 +70,18 @@ INPUT ENCODING processed for backslash escapes, which makes it possible to include any data value in strings that are passed to the library for matching. For patterns, there is a facility for specifying some or all of the 8-bit - input characters as hexadecimal pairs, which makes it possible to - include binary zeros. + input characters as hexadecimal pairs, which makes it possible to in- + clude binary zeros. Input for the 16-bit and 32-bit libraries When testing the 16-bit or 32-bit libraries, there is a need to be able to generate character code points greater than 255 in the strings that are passed to the library. For subject lines, backslash escapes can be - used. In addition, when the utf modifier (see "Setting compilation - options" below) is set, the pattern and any following subject lines are - interpreted as UTF-8 strings and translated to UTF-16 or UTF-32 as - appropriate. + used. In addition, when the utf modifier (see "Setting compilation op- + tions" below) is set, the pattern and any following subject lines are + interpreted as UTF-8 strings and translated to UTF-16 or UTF-32 as ap- + propriate. For non-UTF testing of wide characters, the utf8_input modifier can be used. This is mutually exclusive with utf, and is allowed only in @@ -121,8 +121,8 @@ COMMAND LINE OPTIONS piled. -AC As for -ac, but in addition behave as if each subject line - has the callout_extra modifier, that is, show additional - information from callouts. + has the callout_extra modifier, that is, show additional in- + formation from callouts. -b Behave as if each pattern has the fullbincode modifier; the full internal binary form of the pattern is output after com- @@ -130,9 +130,9 @@ COMMAND LINE OPTIONS -C Output the version number of the PCRE2 library, and all available information about the optional features that are - included, and then exit with zero exit code. All other - options are ignored. If both -C and -LM are present, which- - ever is first is recognized. + included, and then exit with zero exit code. All other op- + tions are ignored. If both -C and -LM are present, whichever + is first is recognized. -C option Output information about a specific build-time option, then exit. This functionality is intended for use in scripts such @@ -190,15 +190,20 @@ COMMAND LINE OPTIONS successful compilation, each pattern is passed to the just- in-time compiler, if available. + -jitfast Behave as if each pattern line has the jitfast modifier; af- + ter successful compilation, each pattern is passed to the + just-in-time compiler, if available, and each subject line is + passed directly to the JIT matcher via its "fast path". + -jitverify - Behave as if each pattern line has the jitverify modifier; - after successful compilation, each pattern is passed to the - just-in-time compiler, if available, and the use of JIT is - verified. + Behave as if each pattern line has the jitverify modifier; + after successful compilation, each pattern is passed to the + just-in-time compiler, if available, and the use of JIT for + matching is verified. -LM List modifiers: write a list of available pattern and subject - modifiers to the standard output, then exit with zero exit - code. All other options are ignored. If both -C and -LM are + modifiers to the standard output, then exit with zero exit + code. All other options are ignored. If both -C and -LM are present, whichever is first is recognized. -pattern modifier-list @@ -207,25 +212,25 @@ COMMAND LINE OPTIONS -q Do not output the version number of pcre2test at the start of execution. - -S size On Unix-like systems, set the size of the run-time stack to + -S size On Unix-like systems, set the size of the run-time stack to size mebibytes (units of 1024*1024 bytes). -subject modifier-list Behave as if each subject line contains the given modifiers. - -t Run each compile and match many times with a timer, and out- - put the resulting times per compile or match. When JIT is - used, separate times are given for the initial compile and - the JIT compile. You can control the number of iterations - that are used for timing by following -t with a number (as a - separate item on the command line). For example, "-t 1000" + -t Run each compile and match many times with a timer, and out- + put the resulting times per compile or match. When JIT is + used, separate times are given for the initial compile and + the JIT compile. You can control the number of iterations + that are used for timing by following -t with a number (as a + separate item on the command line). For example, "-t 1000" iterates 1000 times. The default is to iterate 500,000 times. -tm This is like -t except that it times only the matching phase, not the compile phase. - -T -TM These behave like -t and -tm, but in addition, at the end of - a run, the total times for all compiles and matches are out- + -T -TM These behave like -t and -tm, but in addition, at the end of + a run, the total times for all compiles and matches are out- put. -version Output the PCRE2 version number and then exit. @@ -233,26 +238,27 @@ COMMAND LINE OPTIONS DESCRIPTION - If pcre2test is given two filename arguments, it reads from the first + If pcre2test is given two filename arguments, it reads from the first and writes to the second. If the first name is "-", input is taken from - the standard input. If pcre2test is given only one argument, it reads + the standard input. If pcre2test is given only one argument, it reads from that file and writes to stdout. Otherwise, it reads from stdin and writes to stdout. - When pcre2test is built, a configuration option can specify that it - should be linked with the libreadline or libedit library. When this is - done, if the input is from a terminal, it is read using the readline() + When pcre2test is built, a configuration option can specify that it + should be linked with the libreadline or libedit library. When this is + done, if the input is from a terminal, it is read using the readline() function. This provides line-editing and history facilities. The output from the -help option states whether or not readline() will be used. - The program handles any number of tests, each of which consists of a - set of input lines. Each set starts with a regular expression pattern, + The program handles any number of tests, each of which consists of a + set of input lines. Each set starts with a regular expression pattern, followed by any number of subject lines to be matched against that pat- tern. In between sets of test data, command lines that begin with # may appear. This file format, with some restrictions, can also be processed - by the perltest.sh script that is distributed with PCRE2 as a means of + by the perltest.sh script that is distributed with PCRE2 as a means of checking that the behaviour of PCRE2 and Perl is the same. For a speci- - fication of perltest.sh, see the comments near its beginning. + fication of perltest.sh, see the comments near its beginning. See also + the #perltest command below. When the input is a terminal, pcre2test prompts for each line of input, using "re>" to prompt for regular expression patterns, and "data>" to @@ -269,8 +275,8 @@ DESCRIPTION supply them explicitly. An empty line or the end of the file signals the end of the subject - lines for a test, at which point a new pattern or command line is - expected if there is still input to be read. + lines for a test, at which point a new pattern or command line is ex- + pected if there is still input to be read. COMMAND LINES @@ -304,6 +310,12 @@ COMMAND LINES as described in the section entitled "Saving and restoring compiled patterns" below. + #loadtables <filename> + + This command is used to load a set of binary character tables that can + be accessed by the tables=3 qualifier. Such tables can be created by + the pcre2_dftables program with the -b option. + #newline_default [<newline-list>] When PCRE2 is built, a default newline convention can be specified. @@ -311,8 +323,8 @@ COMMAND LINES as indicating a newline in a pattern or subject string. The default can be overridden when a pattern is compiled. The standard test files con- tain tests of various newline conventions, but the majority of the - tests expect a single linefeed to be recognized as a newline by - default. Without special action the tests would fail when PCRE2 is com- + tests expect a single linefeed to be recognized as a newline by de- + fault. Without special action the tests would fail when PCRE2 is com- piled with either CR or CRLF as the default newline. The #newline_default command specifies a list of newline types that are @@ -323,14 +335,14 @@ COMMAND LINES If the default newline is in the list, this command has no effect. Oth- erwise, except when testing the POSIX API, a newline modifier that - specifies the first newline convention in the list (LF in the above - example) is added to any pattern that does not already have a newline + specifies the first newline convention in the list (LF in the above ex- + ample) is added to any pattern that does not already have a newline modifier. If the newline list is empty, the feature is turned off. This command is present in a number of the standard test input files. - When the POSIX API is being tested there is no way to override the - default newline convention, though it is possible to set the newline - convention from within the pattern. A warning is given if the posix or + When the POSIX API is being tested there is no way to override the de- + fault newline convention, though it is possible to set the newline con- + vention from within the pattern. A warning is given if the posix or posix_nosub modifier is used when #newline_default would set a default for the non-POSIX API. @@ -341,14 +353,20 @@ COMMAND LINES #perltest - The appearance of this line causes all subsequent modifier settings to - be checked for compatibility with the perltest.sh script, which is used - to confirm that Perl gives the same results as PCRE2. Also, apart from - comment lines, #pattern commands, and #subject commands that set or - unset "mark", no command lines are permitted, because they and many of - the modifiers are specific to pcre2test, and should not be used in test - files that are also processed by perltest.sh. The #perltest command - helps detect tests that are accidentally put in the wrong file. + This line is used in test files that can also be processed by perl- + test.sh to confirm that Perl gives the same results as PCRE2. Subse- + quent tests are checked for the use of pcre2test features that are in- + compatible with the perltest.sh script. + + Patterns must use '/' as their delimiter, and only certain modifiers + are supported. Comment lines, #pattern commands, and #subject commands + that set or unset "mark" are recognized and acted on. The #perltest, + #forbid_utf, and #newline_default commands, which are needed in the + relevant pcre2test files, are silently ignored. All other command lines + are ignored, but give a warning message. The #perltest command helps + detect tests that are accidentally put in the wrong file or use the + wrong delimiter. For more details of the perltest.sh script see the + comments it contains. #pop [<modifiers>] #popcopy [<modifiers>] @@ -376,8 +394,8 @@ MODIFIER SYNTAX list are separated by commas followed by optional white space. Trailing whitespace in a modifier list is ignored. Some modifiers may be given for both patterns and subject lines, whereas others are valid only for - one or the other. Each modifier has a long name, for example - "anchored", and some of them must be followed by an equals sign and a + one or the other. Each modifier has a long name, for example "an- + chored", and some of them must be followed by an equals sign and a value, for example, "offset=12". Values cannot contain comma charac- ters, but may contain spaces. Modifiers that do not take values may be preceded by a minus sign to turn off a previous setting. @@ -498,8 +516,8 @@ SUBJECT LINE SYNTAX \= This is a comment. abc\= This is an invalid modifier list. - A backslash followed by any other non-alphanumeric character just - escapes that character. A backslash followed by anything else causes an + A backslash followed by any other non-alphanumeric character just es- + capes that character. A backslash followed by anything else causes an error. However, if the very last character in the line is a backslash (and there is no modifier list), it is ignored. This gives a way of passing an empty line as data, since a real empty line terminates the @@ -523,13 +541,13 @@ PATTERN MODIFIERS The following modifiers set options for pcre2_compile(). Most of them set bits in the options argument of that function, but those whose names start with PCRE2_EXTRA are additional options that are set in the - compile context. For the main options, there are some single-letter - abbreviations that are the same as Perl options. There is special han- + compile context. For the main options, there are some single-letter ab- + breviations that are the same as Perl options. There is special han- dling for /x: if a second x is present, PCRE2_EXTENDED is converted - into PCRE2_EXTENDED_MORE as in Perl. A third appearance adds - PCRE2_EXTENDED as well, though this makes no difference to the way - pcre2_compile() behaves. See pcre2api for a description of the effects - of these options. + into PCRE2_EXTENDED_MORE as in Perl. A third appearance adds PCRE2_EX- + TENDED as well, though this makes no difference to the way pcre2_com- + pile() behaves. See pcre2api for a description of the effects of these + options. allow_empty_class set PCRE2_ALLOW_EMPTY_CLASS allow_surrogate_escapes set PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES @@ -551,6 +569,7 @@ PATTERN MODIFIERS firstline set PCRE2_FIRSTLINE literal set PCRE2_LITERAL match_line set PCRE2_EXTRA_MATCH_LINE + match_invalid_utf set PCRE2_MATCH_INVALID_UTF match_unset_backref set PCRE2_MATCH_UNSET_BACKREF match_word set PCRE2_EXTRA_MATCH_WORD /m multiline set PCRE2_MULTILINE @@ -576,9 +595,9 @@ PATTERN MODIFIERS Setting compilation controls - The following modifiers affect the compilation process or request - information about the pattern. There are single-letter abbreviations - for some that are heavily used in the test files. + The following modifiers affect the compilation process or request in- + formation about the pattern. There are single-letter abbreviations for + some that are heavily used in the test files. bsr=[anycrlf|unicode] specify \R handling /B bincode show binary code without lengths @@ -607,7 +626,7 @@ PATTERN MODIFIERS pushcopy push a copy onto the stack stackguard=<number> test the stackguard feature subject_literal treat all subject lines as literal - tables=[0|1|2] select internal tables + tables=[0|1|2|3] select internal tables use_length do not zero-terminate the pattern utf8_input treat input as UTF-8 @@ -669,7 +688,9 @@ PATTERN MODIFIERS as "starting code units". "Last code unit" is the last literal code unit that must be present in any match. This is not necessarily the last character. These lines are omitted if no starting or ending code - units are recorded. + units are recorded. The subject length line is omitted when + no_start_optimize is set because the minimum length is not calculated + when it can never be used. The framesize modifier shows the size, in bytes, of the storage frames used by pcre2_match() for handling backtracking. The size depends on @@ -714,8 +735,8 @@ PATTERN MODIFIERS minated strings but can be passed by length instead of being zero-ter- minated. The use_length modifier causes this to happen. Using a length happens automatically (whether or not use_length is set) when hex is - set, because patterns specified in hexadecimal may contain binary - zeros. + set, because patterns specified in hexadecimal may contain binary ze- + ros. If hex or use_length is used with the POSIX wrapper API (see "Using the POSIX wrapper API" below), the REG_PEND extension is used to pass the @@ -767,8 +788,8 @@ PATTERN MODIFIERS partial modifier in "Subject Modifiers" below for details of how these options are specified for each match attempt. - JIT compilation is requested by the jit pattern modifier, which may - optionally be followed by an equals sign and a number in the range 0 to + JIT compilation is requested by the jit pattern modifier, which may op- + tionally be followed by an equals sign and a number in the range 0 to 7. The three bits that make up the number specify which of the three JIT operating modes are to be compiled: @@ -796,8 +817,8 @@ PATTERN MODIFIERS none was compiled for non-partial matching. If JIT compilation is successful, the compiled JIT code will automati- - cally be used when an appropriate type of match is run, except when - incompatible run-time options are specified. For more details, see the + cally be used when an appropriate type of match is run, except when in- + compatible run-time options are specified. For more details, see the pcre2jit documentation. See also the jitstack modifier below for a way of setting the size of the JIT stack. @@ -844,8 +865,8 @@ PATTERN MODIFIERS Limiting nested parentheses The parens_nest_limit modifier sets a limit on the depth of nested - parentheses in a pattern. Breaching the limit causes a compilation - error. The default for the library is set when PCRE2 is built, but + parentheses in a pattern. Breaching the limit causes a compilation er- + ror. The default for the library is set when PCRE2 is built, but pcre2test sets its own default of 220, which is required for running the standard test suite. @@ -883,13 +904,13 @@ PATTERN MODIFIERS buffer is too small for the error message. If this modifier has not been set, a large buffer is used. - The aftertext and allaftertext subject modifiers work as described - below. All other modifiers are either ignored, with a warning message, - or cause an error. + The aftertext and allaftertext subject modifiers work as described be- + low. All other modifiers are either ignored, with a warning message, or + cause an error. - The pattern is passed to regcomp() as a zero-terminated string by - default, but if the use_length or hex modifiers are set, the REG_PEND - extension is used to pass it by length. + The pattern is passed to regcomp() as a zero-terminated string by de- + fault, but if the use_length or hex modifiers are set, the REG_PEND ex- + tension is used to pass it by length. Testing the stack guard feature @@ -906,80 +927,85 @@ PATTERN MODIFIERS Using alternative character tables The value specified for the tables modifier must be one of the digits - 0, 1, or 2. It causes a specific set of built-in character tables to be - passed to pcre2_compile(). This is used in the PCRE2 tests to check be- - haviour with different character tables. The digit specifies the tables - as follows: + 0, 1, 2, or 3. It causes a specific set of built-in character tables to + be passed to pcre2_compile(). This is used in the PCRE2 tests to check + behaviour with different character tables. The digit specifies the ta- + bles as follows: 0 do not pass any special character tables 1 the default ASCII tables, as distributed in pcre2_chartables.c.dist 2 a set of tables defining ISO 8859 characters + 3 a set of tables loaded by the #loadtables command - In table 2, some characters whose codes are greater than 128 are iden- - tified as letters, digits, spaces, etc. Setting alternate character - tables and a locale are mutually exclusive. + In tables 2, some characters whose codes are greater than 128 are iden- + tified as letters, digits, spaces, etc. Tables 3 can be used only after + a #loadtables command has loaded them from a binary file. Setting al- + ternate character tables and a locale are mutually exclusive. Setting certain match controls The following modifiers are really subject modifiers, and are described - under "Subject Modifiers" below. However, they may be included in a - pattern's modifier list, in which case they are applied to every sub- - ject line that is processed with that pattern. These modifiers do not + under "Subject Modifiers" below. However, they may be included in a + pattern's modifier list, in which case they are applied to every sub- + ject line that is processed with that pattern. These modifiers do not affect the compilation process. - aftertext show text after match - allaftertext show text after captures - allcaptures show all captures - allvector show the entire ovector - allusedtext show all consulted text - altglobal alternative global matching - /g global global matching - jitstack=<n> set size of JIT stack - mark show mark values - replace=<string> specify a replacement string - startchar show starting character when relevant - substitute_callout use substitution callouts - substitute_extended use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n - substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH - substitute_stop=<n> skip substitution number n and greater - substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET - substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY - - These modifiers may not appear in a #pattern command. If you want them + aftertext show text after match + allaftertext show text after captures + allcaptures show all captures + allvector show the entire ovector + allusedtext show all consulted text + altglobal alternative global matching + /g global global matching + jitstack=<n> set size of JIT stack + mark show mark values + replace=<string> specify a replacement string + startchar show starting character when relevant + substitute_callout use substitution callouts + substitute_extended use PCRE2_SUBSTITUTE_EXTENDED + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED + substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution <n> + substitute_stop=<n> skip substitution <n> and following + substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET + substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY + + These modifiers may not appear in a #pattern command. If you want them as defaults, set them in a #subject command. Specifying literal subject lines - If the subject_literal modifier is present on a pattern, all the sub- + If the subject_literal modifier is present on a pattern, all the sub- ject lines that it matches are taken as literal strings, with no inter- - pretation of backslashes. It is not possible to set subject modifiers - on such lines, but any that are set as defaults by a #subject command + pretation of backslashes. It is not possible to set subject modifiers + on such lines, but any that are set as defaults by a #subject command are recognized. Saving a compiled pattern - When a pattern with the push modifier is successfully compiled, it is - pushed onto a stack of compiled patterns, and pcre2test expects the - next line to contain a new pattern (or a command) instead of a subject + When a pattern with the push modifier is successfully compiled, it is + pushed onto a stack of compiled patterns, and pcre2test expects the + next line to contain a new pattern (or a command) instead of a subject line. This facility is used when saving compiled patterns to a file, as - described in the section entitled "Saving and restoring compiled pat- - terns" below. If pushcopy is used instead of push, a copy of the com- - piled pattern is stacked, leaving the original as current, ready to - match the following input lines. This provides a way of testing the - pcre2_code_copy() function. The push and pushcopy modifiers are - incompatible with compilation modifiers such as global that act at - match time. Any that are specified are ignored (for the stacked copy), - with a warning message, except for replace, which causes an error. Note - that jitverify, which is allowed, does not carry through to any subse- - quent matching that uses a stacked pattern. + described in the section entitled "Saving and restoring compiled pat- + terns" below. If pushcopy is used instead of push, a copy of the com- + piled pattern is stacked, leaving the original as current, ready to + match the following input lines. This provides a way of testing the + pcre2_code_copy() function. The push and pushcopy modifiers are in- + compatible with compilation modifiers such as global that act at match + time. Any that are specified are ignored (for the stacked copy), with a + warning message, except for replace, which causes an error. Note that + jitverify, which is allowed, does not carry through to any subsequent + matching that uses a stacked pattern. Testing foreign pattern conversion - The experimental foreign pattern conversion functions in PCRE2 can be - tested by setting the convert modifier. Its argument is a colon-sepa- - rated list of options, which set the equivalent option for the + The experimental foreign pattern conversion functions in PCRE2 can be + tested by setting the convert modifier. Its argument is a colon-sepa- + rated list of options, which set the equivalent option for the pcre2_pattern_convert() function: glob PCRE2_CONVERT_GLOB @@ -991,19 +1017,19 @@ PATTERN MODIFIERS The "unset" value is useful for turning off a default that has been set by a #pattern command. When one of these options is set, the input pat- - tern is passed to pcre2_pattern_convert(). If the conversion is suc- - cessful, the result is reflected in the output and then passed to + tern is passed to pcre2_pattern_convert(). If the conversion is suc- + cessful, the result is reflected in the output and then passed to pcre2_compile(). The normal utf and no_utf_check options, if set, cause - the PCRE2_CONVERT_UTF and PCRE2_CONVERT_NO_UTF_CHECK options to be + the PCRE2_CONVERT_UTF and PCRE2_CONVERT_NO_UTF_CHECK options to be passed to pcre2_pattern_convert(). By default, the conversion function is allowed to allocate a buffer for - its output. However, if the convert_length modifier is set to a value - greater than zero, pcre2test passes a buffer of the given length. This + its output. However, if the convert_length modifier is set to a value + greater than zero, pcre2test passes a buffer of the given length. This makes it possible to test the length check. - The convert_glob_escape and convert_glob_separator modifiers can be - used to specify the escape and separator characters for glob process- + The convert_glob_escape and convert_glob_separator modifiers can be + used to specify the escape and separator characters for glob process- ing, overriding the defaults, which are operating-system dependent. @@ -1014,7 +1040,7 @@ SUBJECT MODIFIERS Setting match options - The following modifiers set options for pcre2_match() or + The following modifiers set options for pcre2_match() or pcre2_dfa_match(). See pcreapi for a description of their effects. anchored set PCRE2_ANCHORED @@ -1030,34 +1056,34 @@ SUBJECT MODIFIERS partial_hard (or ph) set PCRE2_PARTIAL_HARD partial_soft (or ps) set PCRE2_PARTIAL_SOFT - The partial matching modifiers are provided with abbreviations because + The partial matching modifiers are provided with abbreviations because they appear frequently in tests. - If the posix or posix_nosub modifier was present on the pattern, caus- + If the posix or posix_nosub modifier was present on the pattern, caus- ing the POSIX wrapper API to be used, the only option-setting modifiers that have any effect are notbol, notempty, and noteol, causing REG_NOT- - BOL, REG_NOTEMPTY, and REG_NOTEOL, respectively, to be passed to + BOL, REG_NOTEMPTY, and REG_NOTEOL, respectively, to be passed to regexec(). The other modifiers are ignored, with a warning message. - There is one additional modifier that can be used with the POSIX wrap- + There is one additional modifier that can be used with the POSIX wrap- per. It is ignored (with a warning) if used for non-POSIX matching. posix_startend=<n>[:<m>] - This causes the subject string to be passed to regexec() using the - REG_STARTEND option, which uses offsets to specify which part of the - string is searched. If only one number is given, the end offset is - passed as the end of the subject string. For more detail of REG_STAR- - TEND, see the pcre2posix documentation. If the subject string contains - binary zeros (coded as escapes such as \x{00} because pcre2test does + This causes the subject string to be passed to regexec() using the + REG_STARTEND option, which uses offsets to specify which part of the + string is searched. If only one number is given, the end offset is + passed as the end of the subject string. For more detail of REG_STAR- + TEND, see the pcre2posix documentation. If the subject string contains + binary zeros (coded as escapes such as \x{00} because pcre2test does not support actual binary zeros in its input), you must use posix_star- tend to specify its length. Setting match controls - The following modifiers affect the matching process or request addi- - tional information. Some of them may also be specified on a pattern - line (see above), in which case they apply to every subject line that + The following modifiers affect the matching process or request addi- + tional information. Some of them may also be specified on a pattern + line (see above), in which case they apply to every subject line that is matched against that pattern. aftertext show text after match @@ -1095,55 +1121,62 @@ SUBJECT MODIFIERS startoffset=<n> same as offset=<n> substitute_callout use substitution callouts substitute_extedded use PCRE2_SUBSTITUTE_EXTENDED - substitute_skip=<n> skip substitution number n + substitute_literal use PCRE2_SUBSTITUTE_LITERAL + substitute_matched use PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length use PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only use PCRE2_SUBSTITUTE_REPLACEMENT_ONLY + substitute_skip=<n> skip substitution number n substitute_stop=<n> skip substitution number n and greater substitute_unknown_unset use PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty use PCRE2_SUBSTITUTE_UNSET_EMPTY zero_terminate pass the subject as zero-terminated The effects of these modifiers are described in the following sections. - When matching via the POSIX wrapper API, the aftertext, allaftertext, - and ovector subject modifiers work as described below. All other modi- + When matching via the POSIX wrapper API, the aftertext, allaftertext, + and ovector subject modifiers work as described below. All other modi- fiers are either ignored, with a warning message, or cause an error. Showing more text - The aftertext modifier requests that as well as outputting the part of + The aftertext modifier requests that as well as outputting the part of the subject string that matched the entire pattern, pcre2test should in addition output the remainder of the subject string. This is useful for tests where the subject contains multiple copies of the same substring. - The allaftertext modifier requests the same action for captured sub- + The allaftertext modifier requests the same action for captured sub- strings as well as the main matched substring. In each case the remain- der is output on the following line with a plus character following the capture number. - The allusedtext modifier requests that all the text that was consulted - during a successful pattern match by the interpreter should be shown. - This feature is not supported for JIT matching, and if requested with - JIT it is ignored (with a warning message). Setting this modifier - affects the output if there is a lookbehind at the start of a match, or - a lookahead at the end, or if \K is used in the pattern. Characters - that precede or follow the start and end of the actual match are indi- - cated in the output by '<' or '>' characters underneath them. Here is - an example: + The allusedtext modifier requests that all the text that was consulted + during a successful pattern match by the interpreter should be shown, + for both full and partial matches. This feature is not supported for + JIT matching, and if requested with JIT it is ignored (with a warning + message). Setting this modifier affects the output if there is a look- + behind at the start of a match, or, for a complete match, a lookahead + at the end, or if \K is used in the pattern. Characters that precede or + follow the start and end of the actual match are indicated in the out- + put by '<' or '>' characters underneath them. Here is an example: re> /(?<=pqr)abc(?=xyz)/ data> 123pqrabcxyz456\=allusedtext 0: pqrabcxyz <<< >>> + data> 123pqrabcxy\=ph,allusedtext + Partial match: pqrabcxy + <<< - This shows that the matched string is "abc", with the preceding and - following strings "pqr" and "xyz" having been consulted during the - match (when processing the assertions). + The first, complete match shows that the matched string is "abc", with + the preceding and following strings "pqr" and "xyz" having been con- + sulted during the match (when processing the assertions). The partial + match can indicate only the preceding string. The startchar modifier requests that the starting character for the match be indicated, if it is different to the start of the matched string. The only time when this occurs is when \K has been processed as part of the match. In this situation, the output for the matched string is displayed from the starting character instead of from the match - point, with circumflex characters under the earlier characters. For - example: + point, with circumflex characters under the earlier characters. For ex- + ample: re> /abc\Kxyz/ data> abcxyz\=startchar @@ -1168,12 +1201,12 @@ SUBJECT MODIFIERS The allvector modifier requests that the entire ovector be shown, what- ever the outcome of the match. Compare allcaptures, which shows only up to the maximum number of capture groups for the pattern, and then only - for a successful complete non-DFA match. This modifier, which acts - after any match result, and also for DFA matching, provides a means of + for a successful complete non-DFA match. This modifier, which acts af- + ter any match result, and also for DFA matching, provides a means of checking that there are no unexpected modifications to ovector fields. Before each match attempt, the ovector is filled with a special value, - and if this is found in both elements of a capturing pair, - "<unchanged>" is output. After a successful match, this applies to all + and if this is found in both elements of a capturing pair, "<un- + changed>" is output. After a successful match, this applies to all groups after the maximum capture group for the pattern. In other cases it applies to the entire ovector. After a partial match, the first two elements are the only ones that should be set. After a DFA match, the @@ -1204,12 +1237,12 @@ SUBJECT MODIFIERS If an empty string is matched, the next match is done with the PCRE2_NOTEMPTY_ATSTART and PCRE2_ANCHORED flags set, in order to search for another, non-empty, match at the same point in the subject. If this - match fails, the start offset is advanced, and the normal match is - retried. This imitates the way Perl handles such cases when using the - /g modifier or the split() function. Normally, the start offset is - advanced by one character, but if the newline convention recognizes - CRLF as a newline, and the current character is CR followed by LF, an - advance of two characters occurs. + match fails, the start offset is advanced, and the normal match is re- + tried. This imitates the way Perl handles such cases when using the /g + modifier or the split() function. Normally, the start offset is ad- + vanced by one character, but if the newline convention recognizes CRLF + as a newline, and the current character is CR followed by LF, an ad- + vance of two characters occurs. Testing substring extraction functions @@ -1237,9 +1270,11 @@ SUBJECT MODIFIERS Testing the substitution function If the replace modifier is set, the pcre2_substitute() function is - called instead of one of the matching functions. Note that replacement - strings cannot contain commas, because a comma signifies the end of a - modifier. This is not thought to be an issue in a test program. + called instead of one of the matching functions (or after one call of + pcre2_match() in the case of PCRE2_SUBSTITUTE_MATCHED). Note that re- + placement strings cannot contain commas, because a comma signifies the + end of a modifier. This is not thought to be an issue in a test pro- + gram. Unlike subject strings, pcre2test does not process replacement strings for escape sequences. In UTF mode, a replacement string is checked to @@ -1253,10 +1288,14 @@ SUBJECT MODIFIERS global PCRE2_SUBSTITUTE_GLOBAL substitute_extended PCRE2_SUBSTITUTE_EXTENDED + substitute_literal PCRE2_SUBSTITUTE_LITERAL + substitute_matched PCRE2_SUBSTITUTE_MATCHED substitute_overflow_length PCRE2_SUBSTITUTE_OVERFLOW_LENGTH + substitute_replacement_only PCRE2_SUBSTITUTE_REPLACEMENT_ONLY substitute_unknown_unset PCRE2_SUBSTITUTE_UNKNOWN_UNSET substitute_unset_empty PCRE2_SUBSTITUTE_UNSET_EMPTY + See the pcre2api documentation for details of these options. After a successful substitution, the modified string is output, pre- ceded by the number of replacements. This may be zero if there were no @@ -1272,8 +1311,8 @@ SUBJECT MODIFIERS than 256 characters) for substitution tests, as fixed-size buffers are used. To make it easy to test for buffer overflow, if the replacement string starts with a number in square brackets, that number is passed - to pcre2_substitute() as the size of the output buffer, with the - replacement string starting at the next character. Here is an example + to pcre2_substitute() as the size of the output buffer, with the re- + placement string starting at the next character. Here is an example that tests the edge case: /abc/ @@ -1282,10 +1321,10 @@ SUBJECT MODIFIERS 123abc123\=replace=[9]XYZ Failed: error -47: no more memory - The default action of pcre2_substitute() is to return - PCRE2_ERROR_NOMEMORY when the output buffer is too small. However, if - the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set (by using the sub- - stitute_overflow_length modifier), pcre2_substitute() continues to go + The default action of pcre2_substitute() is to return PCRE2_ER- + ROR_NOMEMORY when the output buffer is too small. However, if the + PCRE2_SUBSTITUTE_OVERFLOW_LENGTH option is set (by using the substi- + tute_overflow_length modifier), pcre2_substitute() continues to go through the motions of matching and substituting (but not doing any callouts), in order to compute the size of buffer that is required. When this happens, pcre2test shows the required buffer length (which @@ -1320,8 +1359,8 @@ SUBJECT MODIFIERS Then are listed the offsets of the old substring, its contents, and the same for the replacement. - By default, the substitution callout function returns zero, which - accepts the replacement and causes matching to continue if /g was used. + By default, the substitution callout function returns zero, which ac- + cepts the replacement and causes matching to continue if /g was used. Two further modifiers can be used to test other return values. If sub- stitute_skip is set to a value greater than zero the callout function returns +1 for the match of that number, and similarly substitute_stop @@ -1408,8 +1447,8 @@ SUBJECT MODIFIERS The memory modifier causes pcre2test to log the sizes of all heap mem- ory allocation and freeing calls that occur during a call to - pcre2_match() or pcre2_dfa_match(). These occur only when a match - requires a bigger vector than the default for remembering backtracking + pcre2_match() or pcre2_dfa_match(). These occur only when a match re- + quires a bigger vector than the default for remembering backtracking points (pcre2_match()) or for internal workspace (pcre2_dfa_match()). In many cases there will be no heap memory used and therefore no addi- tional output. No heap memory is allocated during matching with JIT, so @@ -1432,9 +1471,9 @@ SUBJECT MODIFIERS Setting the size of the output vector - The ovector modifier applies only to the subject line in which it - appears, though of course it can also be used to set a default in a - #subject command. It specifies the number of pairs of offsets that are + The ovector modifier applies only to the subject line in which it ap- + pears, though of course it can also be used to set a default in a #sub- + ject command. It specifies the number of pairs of offsets that are available for storing matching information. The default is 15. A value of zero is useful when testing the POSIX API because it causes @@ -1488,12 +1527,12 @@ DEFAULT OUTPUT FROM pcre2test When a match succeeds, pcre2test outputs the list of captured sub- strings, starting with number 0 for the string that matched the whole - pattern. Otherwise, it outputs "No match" when the return is - PCRE2_ERROR_NOMATCH, or "Partial match:" followed by the partially - matching substring when the return is PCRE2_ERROR_PARTIAL. (Note that - this is the entire substring that was inspected during the partial - match; it may include characters before the actual match start if a - lookbehind assertion, \K, \b, or \B was involved.) + pattern. Otherwise, it outputs "No match" when the return is PCRE2_ER- + ROR_NOMATCH, or "Partial match:" followed by the partially matching + substring when the return is PCRE2_ERROR_PARTIAL. (Note that this is + the entire substring that was inspected during the partial match; it + may include characters before the actual match start if a lookbehind + assertion, \K, \b, or \B was involved.) For any other return, pcre2test outputs the PCRE2 negative error number and a short descriptive phrase. If the error is a failed UTF string @@ -1538,8 +1577,8 @@ DEFAULT OUTPUT FROM pcre2test 0: cat 0+ aract - If global matching is requested, the results of successive matching - attempts are output in sequence, like this: + If global matching is requested, the results of successive matching at- + tempts are output in sequence, like this: re> /\Bi(\w\w)/g data> Mississippi @@ -1577,12 +1616,12 @@ OUTPUT FROM THE ALTERNATIVE MATCHING FUNCTION 2: tan Using the normal matching function on this data finds only "tang". The - longest matching string is always given first (and numbered zero). - After a PCRE2_ERROR_PARTIAL return, the output is "Partial match:", - followed by the partially matching substring. Note that this is the - entire substring that was inspected during the partial match; it may - include characters before the actual match start if a lookbehind asser- - tion, \b, or \B was involved. (\K is not supported for DFA matching.) + longest matching string is always given first (and numbered zero). Af- + ter a PCRE2_ERROR_PARTIAL return, the output is "Partial match:", fol- + lowed by the partially matching substring. Note that this is the entire + substring that was inspected during the partial match; it may include + characters before the actual match start if a lookbehind assertion, \b, + or \B was involved. (\K is not supported for DFA matching.) If global matching is requested, the search for further matches resumes at the end of the longest match. For example: @@ -1609,7 +1648,7 @@ RESTARTING AFTER A PARTIAL MATCH dfa_restart modifier. For example: re> /^\d?\d(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d\d$/ - data> 23ja\=P,dfa + data> 23ja\=ps,dfa Partial match: 23ja data> n05\=dfa,dfa_restart 0: n05 @@ -1635,12 +1674,12 @@ CALLOUTS --->pqrabcdef 0 ^ ^ \d - This output indicates that callout number 0 occurred for a match - attempt starting at the fourth character of the subject string, when - the pointer was at the seventh character, and when the next pattern - item was \d. Just one circumflex is output if the start and current - positions are the same, or if the current position precedes the start - position, which can happen if the callout is in a lookbehind assertion. + This output indicates that callout number 0 occurred for a match at- + tempt starting at the fourth character of the subject string, when the + pointer was at the seventh character, and when the next pattern item + was \d. Just one circumflex is output if the start and current posi- + tions are the same, or if the current position precedes the start posi- + tion, which can happen if the callout is in a lookbehind assertion. Callouts numbered 255 are assumed to be automatic callouts, inserted as a result of the auto_callout pattern modifier. In this case, instead of @@ -1657,8 +1696,8 @@ CALLOUTS 0: E* If a pattern contains (*MARK) items, an additional line is output when- - ever a change of latest mark is passed to the callout function. For - example: + ever a change of latest mark is passed to the callout function. For ex- + ample: re> /a(*MARK:X)bc/auto_callout data> abc @@ -1680,8 +1719,8 @@ CALLOUTS The output for a callout with a string argument is similar, except that instead of outputting a callout number before the position indicators, - the callout string and its offset in the pattern string are output - before the reflection of the subject string, and the subject string is + the callout string and its offset in the pattern string are output be- + fore the reflection of the subject string, and the subject string is reflected for each callout. For example: re> /^ab(?C'first')cd(?C"second")ef/ @@ -1797,9 +1836,9 @@ NON-PRINTING CHARACTERS When pcre2test is outputting text that is a matched part of a subject string, it behaves in the same way, unless a different locale has been - set for the pattern (using the locale modifier). In this case, the - isprint() function is used to distinguish printing and non-printing - characters. + set for the pattern (using the locale modifier). In this case, the is- + print() function is used to distinguish printing and non-printing char- + acters. SAVING AND RESTORING COMPILED PATTERNS @@ -1811,14 +1850,14 @@ SAVING AND RESTORING COMPILED PATTERNS have the same endianness, pointer width and PCRE2_SIZE type. Before compiled patterns can be saved they must be serialized, that is, con- verted to a stream of bytes. A single byte stream may contain any num- - ber of compiled patterns, but they must all use the same character - tables. A single copy of the tables is included in the byte stream (its + ber of compiled patterns, but they must all use the same character ta- + bles. A single copy of the tables is included in the byte stream (its size is 1088 bytes). - The functions whose names begin with pcre2_serialize_ are used for - serializing and de-serializing. They are described in the pcre2serial- - ize documentation. In this section we describe the features of - pcre2test that can be used to test these functions. + The functions whose names begin with pcre2_serialize_ are used for se- + rializing and de-serializing. They are described in the pcre2serialize + documentation. In this section we describe the features of pcre2test + that can be used to test these functions. Note that "serialization" in PCRE2 does not convert compiled patterns to an abstract format like Java or .NET. It just makes a reloadable @@ -1828,8 +1867,8 @@ SAVING AND RESTORING COMPILED PATTERNS piled, it is pushed onto a stack of compiled patterns, and pcre2test expects the next line to contain a new pattern (or command) instead of a subject line. By contrast, the pushcopy modifier causes a copy of the - compiled pattern to be stacked, leaving the original available for - immediate matching. By using push and/or pushcopy, a number of patterns + compiled pattern to be stacked, leaving the original available for im- + mediate matching. By using push and/or pushcopy, a number of patterns can be compiled and retained. These modifiers are incompatible with posix, and control modifiers that act at match time are ignored (with a message) for the stacked patterns. The jitverify modifier applies only @@ -1852,8 +1891,8 @@ SAVING AND RESTORING COMPILED PATTERNS matched with the pattern, terminated as usual by an empty line or end of file. This command may be followed by a modifier list containing only control modifiers that act after a pattern has been compiled. In - particular, hex, posix, posix_nosub, push, and pushcopy are not - allowed, nor are any option-setting modifiers. The JIT modifiers are, + particular, hex, posix, posix_nosub, push, and pushcopy are not al- + lowed, nor are any option-setting modifiers. The JIT modifiers are, however permitted. Here is an example that saves and reloads two pat- terns. @@ -1890,5 +1929,5 @@ AUTHOR REVISION - Last updated: 11 March 2019 - Copyright (c) 1997-2019 University of Cambridge. + Last updated: 14 September 2020 + Copyright (c) 1997-2020 University of Cambridge. diff --git a/dist2/doc/pcre2unicode.3 b/dist2/doc/pcre2unicode.3 index 56eb1eae..055a4ce4 100644 --- a/dist2/doc/pcre2unicode.3 +++ b/dist2/doc/pcre2unicode.3 @@ -1,26 +1,38 @@ -.TH PCRE2UNICODE 3 "06 March 2019" "PCRE2 10.33" +.TH PCRE2UNICODE 3 "23 February 2020" "PCRE2 10.35" .SH NAME PCRE - Perl-compatible regular expressions (revised API) .SH "UNICODE AND UTF SUPPORT" .rs .sp -When PCRE2 is built with Unicode support (which is the default), it has -knowledge of Unicode character properties and can process text strings in -UTF-8, UTF-16, or UTF-32 format (depending on the code unit width). However, by -default, PCRE2 assumes that one code unit is one character. To process a -pattern as a UTF string, where a character may require more than one code unit, -you must call +PCRE2 is normally built with Unicode support, though if you do not need it, you +can build it without, in which case the library will be smaller. With Unicode +support, PCRE2 has knowledge of Unicode character properties and can process +strings of text in UTF-8, UTF-16, and UTF-32 format (depending on the code unit +width), but this is not the default. Unless specifically requested, PCRE2 +treats each code unit in a string as one character. +.P +There are two ways of telling PCRE2 to switch to UTF mode, where characters may +consist of more than one code unit and the range of values is constrained. The +program can call .\" HREF \fBpcre2_compile()\fP .\" -with the PCRE2_UTF option flag, or the pattern must start with the sequence -(*UTF). When either of these is the case, both the pattern and any subject -strings that are matched against it are treated as UTF strings instead of -strings of individual one-code-unit characters. There are also some other -changes to the way characters are handled, as documented below. -.P -If you do not need Unicode support you can build PCRE2 without it, in which -case the library will be smaller. +with the PCRE2_UTF option, or the pattern may start with the sequence (*UTF). +However, the latter facility can be locked out by the PCRE2_NEVER_UTF option. +That is, the programmer can prevent the supplier of the pattern from switching +to UTF mode. +.P +Note that the PCRE2_MATCH_INVALID_UTF option (see +.\" HTML <a href="#matchinvalid"> +.\" </a> +below) +.\" +forces PCRE2_UTF to be set. +.P +In UTF mode, both the pattern and any subject strings that are matched against +it are treated as UTF strings instead of strings of individual one-code-unit +characters. There are also some other changes to the way characters are +handled, as documented below. . . .SH "UNICODE PROPERTY SUPPORT" @@ -55,18 +67,18 @@ also recognized; larger ones can be coded using \eo{...}. .P The escape sequence \eN{U+<hex digits>} is recognized as another way of specifying a Unicode character by code point in a UTF mode. It is not allowed -in non-UTF modes. +in non-UTF mode. .P -In UTF modes, repeat quantifiers apply to complete UTF characters, not to +In UTF mode, repeat quantifiers apply to complete UTF characters, not to individual code units. .P -In UTF modes, the dot metacharacter matches one UTF character instead of a +In UTF mode, the dot metacharacter matches one UTF character instead of a single code unit. .P -In UTF modes, capture group names are not restricted to ASCII, and may contain +In UTF mode, capture group names are not restricted to ASCII, and may contain any Unicode letters and decimal digits, as well as underscore. .P -The escape sequence \eC can be used to match a single code unit in a UTF mode, +The escape sequence \eC can be used to match a single code unit in UTF mode, but its use can lead to some strange effects because it breaks up multi-unit characters (see the description of \eC in the .\" HREF @@ -82,7 +94,7 @@ may consist of more than one code unit. The use of \eC in these modes provokes a match-time error. Also, the JIT optimization does not support \eC in these modes. If JIT optimization is requested for a UTF-8 or UTF-16 pattern that contains \eC, it will not succeed, and so when \fBpcre2_match()\fP is called, -the matching will be carried out by the normal interpretive function. +the matching will be carried out by the interpretive function. .P The character escapes \eb, \eB, \ed, \eD, \es, \eS, \ew, and \eW correctly test characters of any code value, but, by default, the characters that PCRE2 @@ -114,14 +126,16 @@ However, the special horizontal and vertical white space matching escapes (\eh, not PCRE2_UCP is set. . . -.SH "CASE-EQUIVALENCE IN UTF MODES" +.SH "UNICODE CASE-EQUIVALENCE" .rs .sp -Case-insensitive matching in a UTF mode makes use of Unicode properties except -for characters whose code points are less than 128 and that have at most two -case-equivalent values. For these, a direct table lookup is used for speed. A -few Unicode characters such as Greek sigma have more than two code points that -are case-equivalent, and these are treated as such. +If either PCRE2_UTF or PCRE2_UCP is set, upper/lower case processing makes use +of Unicode properties except for characters whose code points are less than 128 +and that have at most two case-equivalent values. For these, a direct table +lookup is used for speed. A few Unicode characters such as Greek sigma have +more than two code points that are case-equivalent, and these are treated +specially. Setting PCRE2_UCP without PCRE2_UTF allows Unicode-style case +processing for non-UTF character encodings such as UCS-2. . . .\" HTML <a name="scriptruns"></a> @@ -231,7 +245,7 @@ adjacent characters. .sp When the PCRE2_UTF option is set, the strings passed as patterns and subjects are (by default) checked for validity on entry to the relevant functions. If an -invalid UTF string is passed, an negative error code is returned. The code unit +invalid UTF string is passed, a negative error code is returned. The code unit offset to the offending character can be extracted from the match data block by calling \fBpcre2_get_startchar()\fP, which is used for this purpose after a UTF error. @@ -244,18 +258,15 @@ PCRE2 assumes that the pattern or subject it is given (respectively) contains only valid UTF code unit sequences. .P If you pass an invalid UTF string when PCRE2_NO_UTF_CHECK is set, the result -is usually undefined and your program may crash or loop indefinitely. There is, -however, one mode of matching that can handle invalid UTF subject strings. This -is matching via the JIT optimization using the PCRE2_JIT_INVALID_UTF option -when calling \fBpcre2_jit_compile()\fP. For details, see the -.\" HREF -\fBpcre2jit\fP -.\" -documentation. +is undefined and your program may crash or loop indefinitely or give incorrect +results. There is, however, one mode of matching that can handle invalid UTF +subject strings. This is enabled by passing PCRE2_MATCH_INVALID_UTF to +\fBpcre2_compile()\fP and is discussed below in the next section. The rest of +this section covers the case when PCRE2_MATCH_INVALID_UTF is not set. .P -Passing PCRE2_NO_UTF_CHECK to \fBpcre2_compile()\fP just disables the check for -the pattern; it does not also apply to subject strings. If you want to disable -the check for a subject string you must pass this same option to +Passing PCRE2_NO_UTF_CHECK to \fBpcre2_compile()\fP just disables the UTF check +for the pattern; it does not also apply to subject strings. If you want to +disable the check for a subject string you must pass this same option to \fBpcre2_match()\fP or \fBpcre2_dfa_match()\fP. .P UTF-16 and UTF-32 strings can indicate their endianness by special code knows @@ -330,7 +341,7 @@ these code points are excluded by RFC 3629. .sp PCRE2_ERROR_UTF8_ERR13 .sp -A 4-byte character has a value greater than 0x10fff; these code points are +A 4-byte character has a value greater than 0x10ffff; these code points are excluded by RFC 3629. .sp PCRE2_ERROR_UTF8_ERR14 @@ -386,6 +397,52 @@ The following negative error codes are given for invalid UTF-32 strings: .sp . . +.\" HTML <a name="matchinvalid"></a> +.SH "MATCHING IN INVALID UTF STRINGS" +.rs +.sp +You can run pattern matches on subject strings that may contain invalid UTF +sequences if you call \fBpcre2_compile()\fP with the PCRE2_MATCH_INVALID_UTF +option. This is supported by \fBpcre2_match()\fP, including JIT matching, but +not by \fBpcre2_dfa_match()\fP. When PCRE2_MATCH_INVALID_UTF is set, it forces +PCRE2_UTF to be set as well. Note, however, that the pattern itself must be a +valid UTF string. +.P +Setting PCRE2_MATCH_INVALID_UTF does not affect what \fBpcre2_compile()\fP +generates, but if \fBpcre2_jit_compile()\fP is subsequently called, it does +generate different code. If JIT is not used, the option affects the behaviour +of the interpretive code in \fBpcre2_match()\fP. When PCRE2_MATCH_INVALID_UTF +is set at compile time, PCRE2_NO_UTF_CHECK is ignored at match time. +.P +In this mode, an invalid code unit sequence in the subject never matches any +pattern item. It does not match dot, it does not match \ep{Any}, it does not +even match negative items such as [^X]. A lookbehind assertion fails if it +encounters an invalid sequence while moving the current point backwards. In +other words, an invalid UTF code unit sequence acts as a barrier which no match +can cross. +.P +You can also think of this as the subject being split up into fragments of +valid UTF, delimited internally by invalid code unit sequences. The pattern is +matched fragment by fragment. The result of a successful match, however, is +given as code unit offsets in the entire subject string in the usual way. There +are a few points to consider: +.P +The internal boundaries are not interpreted as the beginnings or ends of lines +and so do not match circumflex or dollar characters in the pattern. +.P +If \fBpcre2_match()\fP is called with an offset that points to an invalid +UTF-sequence, that sequence is skipped, and the match starts at the next valid +UTF character, or the end of the subject. +.P +At internal fragment boundaries, \eb and \eB behave in the same way as at the +beginning and end of the subject. For example, a sequence such as \ebWORD\eb +would match an instance of WORD that is surrounded by invalid UTF code units. +.P +Using PCRE2_MATCH_INVALID_UTF, an application can run matches on arbitrary +data, knowing that any matched strings that are returned are valid UTF. This +can be useful when searching for UTF text in executable or other binary files. +. +. .SH AUTHOR .rs .sp @@ -400,6 +457,6 @@ Cambridge, England. .rs .sp .nf -Last updated: 06 March 2019 -Copyright (c) 1997-2019 University of Cambridge. +Last updated: 23 February 2020 +Copyright (c) 1997-2020 University of Cambridge. .fi diff --git a/dist2/install-sh b/dist2/install-sh index 8175c640..20d8b2ea 100755 --- a/dist2/install-sh +++ b/dist2/install-sh @@ -451,7 +451,18 @@ do trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 # Copy the file name to the temp name. - (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + (umask $cp_umask && + { test -z "$stripcmd" || { + # Create $dsttmp read-write so that cp doesn't create it read-only, + # which would cause strip to fail. + if test -z "$doit"; then + : >"$dsttmp" # No need to fork-exec 'touch'. + else + $doit touch "$dsttmp" + fi + } + } && + $doit_exec $cpprog "$src" "$dsttmp") && # and set any options; do chmod last to preserve setuid bits. # diff --git a/dist2/libpcre2-16.pc.in b/dist2/libpcre2-16.pc.in index 978040df..bacb4665 100644 --- a/dist2/libpcre2-16.pc.in +++ b/dist2/libpcre2-16.pc.in @@ -8,6 +8,6 @@ includedir=@includedir@ Name: libpcre2-16 Description: PCRE2 - Perl compatible regular expressions C library (2nd API) with 16 bit character support Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre2-16 +Libs: -L${libdir} -lpcre2-16@LIB_POSTFIX@ Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ Cflags: -I${includedir} @PCRE2_STATIC_CFLAG@ diff --git a/dist2/libpcre2-32.pc.in b/dist2/libpcre2-32.pc.in index d8fb1871..06241f06 100644 --- a/dist2/libpcre2-32.pc.in +++ b/dist2/libpcre2-32.pc.in @@ -8,6 +8,6 @@ includedir=@includedir@ Name: libpcre2-32 Description: PCRE2 - Perl compatible regular expressions C library (2nd API) with 32 bit character support Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre2-32 +Libs: -L${libdir} -lpcre2-32@LIB_POSTFIX@ Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ Cflags: -I${includedir} @PCRE2_STATIC_CFLAG@ diff --git a/dist2/libpcre2-8.pc.in b/dist2/libpcre2-8.pc.in index 5c872d0b..246bb9ea 100644 --- a/dist2/libpcre2-8.pc.in +++ b/dist2/libpcre2-8.pc.in @@ -8,6 +8,6 @@ includedir=@includedir@ Name: libpcre2-8 Description: PCRE2 - Perl compatible regular expressions C library (2nd API) with 8 bit character support Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre2-8 +Libs: -L${libdir} -lpcre2-8@LIB_POSTFIX@ Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ Cflags: -I${includedir} @PCRE2_STATIC_CFLAG@ diff --git a/dist2/libpcre2-posix.pc.in b/dist2/libpcre2-posix.pc.in index 96415558..758c3068 100644 --- a/dist2/libpcre2-posix.pc.in +++ b/dist2/libpcre2-posix.pc.in @@ -8,6 +8,6 @@ includedir=@includedir@ Name: libpcre2-posix Description: Posix compatible interface to libpcre2-8 Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre2-posix +Libs: -L${libdir} -lpcre2-posix@LIB_POSTFIX@ Cflags: -I${includedir} @PCRE2_STATIC_CFLAG@ Requires.private: libpcre2-8 diff --git a/dist2/ltmain.sh b/dist2/ltmain.sh index d3ab94d6..48cea9b0 100644 --- a/dist2/ltmain.sh +++ b/dist2/ltmain.sh @@ -2,7 +2,7 @@ ## DO NOT EDIT - This file generated from ./build-aux/ltmain.in ## by inline-source v2018-07-24.06 -# libtool (GNU libtool) 2.4.6.42-b88ce +# libtool (GNU libtool) 2.4.6.42-b88ce-dirty # Provide generalized library-building support services. # Written by Gordon Matzigkeit <gord@gnu.ai.mit.edu>, 1996 @@ -31,7 +31,7 @@ PROGRAM=libtool PACKAGE=libtool -VERSION=2.4.6.42-b88ce +VERSION=2.4.6.42-b88ce-dirty package_revision=2.4.6.42 @@ -2176,7 +2176,7 @@ func_version () # End: # Set a version string. -scriptversion='(GNU libtool) 2.4.6.42-b88ce' +scriptversion='(GNU libtool) 2.4.6.42-b88ce-dirty' # func_echo ARG... @@ -2267,7 +2267,7 @@ include the following information: compiler: $LTCC compiler flags: $LTCFLAGS linker: $LD (gnu? $with_gnu_ld) - version: $progname (GNU libtool) 2.4.6.42-b88ce + version: $progname (GNU libtool) 2.4.6.42-b88ce-dirty automake: `($AUTOMAKE --version) 2>/dev/null |$SED 1q` autoconf: `($AUTOCONF --version) 2>/dev/null |$SED 1q` diff --git a/dist2/m4/libtool.m4 b/dist2/m4/libtool.m4 index b55a6e57..2b73e384 100644 --- a/dist2/m4/libtool.m4 +++ b/dist2/m4/libtool.m4 @@ -728,7 +728,6 @@ _LT_CONFIG_SAVE_COMMANDS([ cat <<_LT_EOF >> "$cfgfile" #! $SHELL # Generated automatically by $as_me ($PACKAGE) $VERSION -# Libtool was configured on host `(hostname || uname -n) 2>/dev/null | sed 1q`: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # Provide generalized library-building support services. diff --git a/dist2/m4/ltversion.m4 b/dist2/m4/ltversion.m4 index 86b2ad72..7f9a3ada 100644 --- a/dist2/m4/ltversion.m4 +++ b/dist2/m4/ltversion.m4 @@ -12,11 +12,11 @@ # serial 4221 ltversion.m4 # This file is part of GNU Libtool -m4_define([LT_PACKAGE_VERSION], [2.4.6.42-b88ce]) +m4_define([LT_PACKAGE_VERSION], [2.4.6.42-b88ce-dirty]) m4_define([LT_PACKAGE_REVISION], [2.4.6.42]) AC_DEFUN([LTVERSION_VERSION], -[macro_version='2.4.6.42-b88ce' +[macro_version='2.4.6.42-b88ce-dirty' macro_revision='2.4.6.42' _LT_DECL(, macro_version, 0, [Which release of libtool.m4 was used?]) _LT_DECL(, macro_revision, 0) diff --git a/dist2/missing b/dist2/missing index 625aeb11..8d0eaad2 100755 --- a/dist2/missing +++ b/dist2/missing @@ -3,7 +3,7 @@ scriptversion=2018-03-07.03; # UTC -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2020 Free Software Foundation, Inc. # Originally written by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996. # This program is free software; you can redistribute it and/or modify diff --git a/dist2/pcre2-config.in b/dist2/pcre2-config.in index 74271c08..bacea876 100644 --- a/dist2/pcre2-config.in +++ b/dist2/pcre2-config.in @@ -86,28 +86,28 @@ while test $# -gt 0; do ;; --libs-posix) if test @enable_pcre2_8@ = yes ; then - echo $libS$libR -lpcre2-posix -lpcre2-8 + echo $libS$libR -lpcre2-posix@LIB_POSTFIX@ -lpcre2-8@LIB_POSTFIX@ else echo "${usage}" 1>&2 fi ;; --libs8) if test @enable_pcre2_8@ = yes ; then - echo $libS$libR -lpcre2-8 + echo $libS$libR -lpcre2-8@LIB_POSTFIX@ else echo "${usage}" 1>&2 fi ;; --libs16) if test @enable_pcre2_16@ = yes ; then - echo $libS$libR -lpcre2-16 + echo $libS$libR -lpcre2-16@LIB_POSTFIX@ else echo "${usage}" 1>&2 fi ;; --libs32) if test @enable_pcre2_32@ = yes ; then - echo $libS$libR -lpcre2-32 + echo $libS$libR -lpcre2-32@LIB_POSTFIX@ else echo "${usage}" 1>&2 fi diff --git a/dist2/perltest.sh b/dist2/perltest.sh index 406a14e0..31406c52 100755 --- a/dist2/perltest.sh +++ b/dist2/perltest.sh @@ -68,6 +68,9 @@ fi # #newline_default commands, which are needed in the relevant pcre2test files, # are ignored. Any other #-command is ignored, with a warning message. # +# The pattern lines should use only / as the delimiter. The other characters +# that pcre2test supports cause problems with this script. +# # The data lines must not have any pcre2test modifiers. Unless # "subject_literal" is on the pattern, data lines are processed as # Perl double-quoted strings, so if they contain " $ or @ characters, these @@ -140,7 +143,7 @@ if (@ARGV > 1) } else { $outfile = "STDOUT"; } -printf($outfile "Perl $] Regular Expressions\n\n"); +printf($outfile "Perl $^V\n\n"); $extra_modifiers = ""; $default_show_mark = 0; diff --git a/dist2/src/config.h.generic b/dist2/src/config.h.generic index 25d45eeb..10f41047 100644 --- a/dist2/src/config.h.generic +++ b/dist2/src/config.h.generic @@ -52,6 +52,9 @@ sure both macros are undefined; an emulation function will then be used. */ LF does in an ASCII/Unicode environment. */ /* #undef EBCDIC_NL25 */ +/* Define this if your compiler supports __attribute__((uninitialized)) */ +/* #undef HAVE_ATTRIBUTE_UNINITIALIZED */ + /* Define to 1 if you have the `bcopy' function. */ /* #undef HAVE_BCOPY */ @@ -76,6 +79,9 @@ sure both macros are undefined; an emulation function will then be used. */ /* Define to 1 if you have the <limits.h> header file. */ /* #undef HAVE_LIMITS_H */ +/* Define to 1 if you have the `memfd_create' function. */ +/* #undef HAVE_MEMFD_CREATE */ + /* Define to 1 if you have the `memmove' function. */ /* #undef HAVE_MEMMOVE */ @@ -218,7 +224,7 @@ sure both macros are undefined; an emulation function will then be used. */ #define PACKAGE_NAME "PCRE2" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PCRE2 10.33" +#define PACKAGE_STRING "PCRE2 10.36" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "pcre2" @@ -227,7 +233,7 @@ sure both macros are undefined; an emulation function will then be used. */ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "10.33" +#define PACKAGE_VERSION "10.36" /* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested parentheses (of any kind) in a pattern. This limits the amount of system @@ -352,7 +358,7 @@ sure both macros are undefined; an emulation function will then be used. */ #endif /* Version number of package */ -#define VERSION "10.33" +#define VERSION "10.36" /* Define to 1 if on MINIX. */ /* #undef _MINIX */ diff --git a/dist2/src/config.h.in b/dist2/src/config.h.in index 5406da0a..d42cc005 100644 --- a/dist2/src/config.h.in +++ b/dist2/src/config.h.in @@ -79,6 +79,9 @@ sure both macros are undefined; an emulation function will then be used. */ /* Define to 1 if you have the <limits.h> header file. */ #undef HAVE_LIMITS_H +/* Define to 1 if you have the `memfd_create' function. */ +#undef HAVE_MEMFD_CREATE + /* Define to 1 if you have the `memmove' function. */ #undef HAVE_MEMMOVE diff --git a/dist2/src/pcre2.h.generic b/dist2/src/pcre2.h.generic index 102b5d91..f204ec81 100644 --- a/dist2/src/pcre2.h.generic +++ b/dist2/src/pcre2.h.generic @@ -5,7 +5,7 @@ /* This is the public header file for the PCRE library, second API, to be #included by applications that call PCRE2 functions. - Copyright (c) 2016-2018 University of Cambridge + Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -42,9 +42,9 @@ POSSIBILITY OF SUCH DAMAGE. /* The current PCRE version information. */ #define PCRE2_MAJOR 10 -#define PCRE2_MINOR 33 +#define PCRE2_MINOR 36 #define PCRE2_PRERELEASE -#define PCRE2_DATE 2019-04-16 +#define PCRE2_DATE 2020-12-04 /* When an application links to a PCRE DLL in Windows, the symbols that are imported have to be identified as such. When building PCRE2, the appropriate @@ -142,6 +142,7 @@ D is inspected during pcre2_dfa_match() execution #define PCRE2_USE_OFFSET_LIMIT 0x00800000u /* J M D */ #define PCRE2_EXTENDED_MORE 0x01000000u /* C */ #define PCRE2_LITERAL 0x02000000u /* C */ +#define PCRE2_MATCH_INVALID_UTF 0x04000000u /* J M D */ /* An additional compile options word is available in the compile context. */ @@ -180,6 +181,9 @@ pcre2_jit_match() ignores the latter since it bypasses all sanity checks). */ #define PCRE2_SUBSTITUTE_OVERFLOW_LENGTH 0x00001000u /* pcre2_substitute() only */ #define PCRE2_NO_JIT 0x00002000u /* Not for pcre2_dfa_match() */ #define PCRE2_COPY_MATCHED_SUBJECT 0x00004000u +#define PCRE2_SUBSTITUTE_LITERAL 0x00008000u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_MATCHED 0x00010000u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_REPLACEMENT_ONLY 0x00020000u /* pcre2_substitute() only */ /* Options for pcre2_pattern_convert(). */ @@ -305,6 +309,8 @@ pcre2_pattern_convert(). */ #define PCRE2_ERROR_INVALID_HYPHEN_IN_OPTIONS 194 #define PCRE2_ERROR_ALPHA_ASSERTION_UNKNOWN 195 #define PCRE2_ERROR_SCRIPT_RUN_NOT_AVAILABLE 196 +#define PCRE2_ERROR_TOO_MANY_CAPTURES 197 +#define PCRE2_ERROR_CONDITION_ATOMIC_ASSERTION_EXPECTED 198 /* "Expected" matching error codes: no match and partial match. */ @@ -390,6 +396,7 @@ released, the numbers must not be changed. */ #define PCRE2_ERROR_HEAPLIMIT (-63) #define PCRE2_ERROR_CONVERT_SYNTAX (-64) #define PCRE2_ERROR_INTERNAL_DUPMATCH (-65) +#define PCRE2_ERROR_DFA_UINVALID_UTF (-66) /* Request types for pcre2_pattern_info() */ @@ -441,6 +448,7 @@ released, the numbers must not be changed. */ #define PCRE2_CONFIG_HEAPLIMIT 12 #define PCRE2_CONFIG_NEVER_BACKSLASH_C 13 #define PCRE2_CONFIG_COMPILED_WIDTHS 14 +#define PCRE2_CONFIG_TABLES_LENGTH 15 /* Types for code units in patterns and subject strings. */ @@ -580,7 +588,7 @@ PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_bsr(pcre2_compile_context *, uint32_t); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ - pcre2_set_character_tables(pcre2_compile_context *, const unsigned char *); \ + pcre2_set_character_tables(pcre2_compile_context *, const uint8_t *); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_compile_extra_options(pcre2_compile_context *, uint32_t); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ @@ -675,6 +683,8 @@ PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SPTR PCRE2_CALL_CONVENTION \ pcre2_get_mark(pcre2_match_data *); \ +PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ + pcre2_get_match_data_size(pcre2_match_data *); \ PCRE2_EXP_DECL uint32_t PCRE2_CALL_CONVENTION \ pcre2_get_ovector_count(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ @@ -773,7 +783,8 @@ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_get_error_message(int, PCRE2_UCHAR *, PCRE2_SIZE); \ PCRE2_EXP_DECL const uint8_t PCRE2_CALL_CONVENTION \ *pcre2_maketables(pcre2_general_context *); \ - +PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ + pcre2_maketables_free(pcre2_general_context *, const uint8_t *); /* Define macros that generate width-specific names from generic versions. The three-level macro scheme is necessary to get the macros expanded when we want @@ -838,6 +849,7 @@ pcre2_compile are called by application code. */ #define pcre2_general_context_free PCRE2_SUFFIX(pcre2_general_context_free_) #define pcre2_get_error_message PCRE2_SUFFIX(pcre2_get_error_message_) #define pcre2_get_mark PCRE2_SUFFIX(pcre2_get_mark_) +#define pcre2_get_match_data_size PCRE2_SUFFIX(pcre2_get_match_data_size_) #define pcre2_get_ovector_pointer PCRE2_SUFFIX(pcre2_get_ovector_pointer_) #define pcre2_get_ovector_count PCRE2_SUFFIX(pcre2_get_ovector_count_) #define pcre2_get_startchar PCRE2_SUFFIX(pcre2_get_startchar_) @@ -848,6 +860,7 @@ pcre2_compile are called by application code. */ #define pcre2_jit_stack_create PCRE2_SUFFIX(pcre2_jit_stack_create_) #define pcre2_jit_stack_free PCRE2_SUFFIX(pcre2_jit_stack_free_) #define pcre2_maketables PCRE2_SUFFIX(pcre2_maketables_) +#define pcre2_maketables_free PCRE2_SUFFIX(pcre2_maketables_free_) #define pcre2_match PCRE2_SUFFIX(pcre2_match_) #define pcre2_match_context_copy PCRE2_SUFFIX(pcre2_match_context_copy_) #define pcre2_match_context_create PCRE2_SUFFIX(pcre2_match_context_create_) diff --git a/dist2/src/pcre2.h.in b/dist2/src/pcre2.h.in index 9415d702..4fd6a1e3 100644 --- a/dist2/src/pcre2.h.in +++ b/dist2/src/pcre2.h.in @@ -5,7 +5,7 @@ /* This is the public header file for the PCRE library, second API, to be #included by applications that call PCRE2 functions. - Copyright (c) 2016-2018 University of Cambridge + Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -142,6 +142,7 @@ D is inspected during pcre2_dfa_match() execution #define PCRE2_USE_OFFSET_LIMIT 0x00800000u /* J M D */ #define PCRE2_EXTENDED_MORE 0x01000000u /* C */ #define PCRE2_LITERAL 0x02000000u /* C */ +#define PCRE2_MATCH_INVALID_UTF 0x04000000u /* J M D */ /* An additional compile options word is available in the compile context. */ @@ -180,6 +181,9 @@ pcre2_jit_match() ignores the latter since it bypasses all sanity checks). */ #define PCRE2_SUBSTITUTE_OVERFLOW_LENGTH 0x00001000u /* pcre2_substitute() only */ #define PCRE2_NO_JIT 0x00002000u /* Not for pcre2_dfa_match() */ #define PCRE2_COPY_MATCHED_SUBJECT 0x00004000u +#define PCRE2_SUBSTITUTE_LITERAL 0x00008000u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_MATCHED 0x00010000u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_REPLACEMENT_ONLY 0x00020000u /* pcre2_substitute() only */ /* Options for pcre2_pattern_convert(). */ @@ -305,6 +309,8 @@ pcre2_pattern_convert(). */ #define PCRE2_ERROR_INVALID_HYPHEN_IN_OPTIONS 194 #define PCRE2_ERROR_ALPHA_ASSERTION_UNKNOWN 195 #define PCRE2_ERROR_SCRIPT_RUN_NOT_AVAILABLE 196 +#define PCRE2_ERROR_TOO_MANY_CAPTURES 197 +#define PCRE2_ERROR_CONDITION_ATOMIC_ASSERTION_EXPECTED 198 /* "Expected" matching error codes: no match and partial match. */ @@ -390,6 +396,7 @@ released, the numbers must not be changed. */ #define PCRE2_ERROR_HEAPLIMIT (-63) #define PCRE2_ERROR_CONVERT_SYNTAX (-64) #define PCRE2_ERROR_INTERNAL_DUPMATCH (-65) +#define PCRE2_ERROR_DFA_UINVALID_UTF (-66) /* Request types for pcre2_pattern_info() */ @@ -441,6 +448,7 @@ released, the numbers must not be changed. */ #define PCRE2_CONFIG_HEAPLIMIT 12 #define PCRE2_CONFIG_NEVER_BACKSLASH_C 13 #define PCRE2_CONFIG_COMPILED_WIDTHS 14 +#define PCRE2_CONFIG_TABLES_LENGTH 15 /* Types for code units in patterns and subject strings. */ @@ -580,7 +588,7 @@ PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_bsr(pcre2_compile_context *, uint32_t); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ - pcre2_set_character_tables(pcre2_compile_context *, const unsigned char *); \ + pcre2_set_character_tables(pcre2_compile_context *, const uint8_t *); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_compile_extra_options(pcre2_compile_context *, uint32_t); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ @@ -675,6 +683,8 @@ PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ pcre2_match_data_free(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SPTR PCRE2_CALL_CONVENTION \ pcre2_get_mark(pcre2_match_data *); \ +PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ + pcre2_get_match_data_size(pcre2_match_data *); \ PCRE2_EXP_DECL uint32_t PCRE2_CALL_CONVENTION \ pcre2_get_ovector_count(pcre2_match_data *); \ PCRE2_EXP_DECL PCRE2_SIZE PCRE2_CALL_CONVENTION \ @@ -773,7 +783,8 @@ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_get_error_message(int, PCRE2_UCHAR *, PCRE2_SIZE); \ PCRE2_EXP_DECL const uint8_t PCRE2_CALL_CONVENTION \ *pcre2_maketables(pcre2_general_context *); \ - +PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ + pcre2_maketables_free(pcre2_general_context *, const uint8_t *); /* Define macros that generate width-specific names from generic versions. The three-level macro scheme is necessary to get the macros expanded when we want @@ -838,6 +849,7 @@ pcre2_compile are called by application code. */ #define pcre2_general_context_free PCRE2_SUFFIX(pcre2_general_context_free_) #define pcre2_get_error_message PCRE2_SUFFIX(pcre2_get_error_message_) #define pcre2_get_mark PCRE2_SUFFIX(pcre2_get_mark_) +#define pcre2_get_match_data_size PCRE2_SUFFIX(pcre2_get_match_data_size_) #define pcre2_get_ovector_pointer PCRE2_SUFFIX(pcre2_get_ovector_pointer_) #define pcre2_get_ovector_count PCRE2_SUFFIX(pcre2_get_ovector_count_) #define pcre2_get_startchar PCRE2_SUFFIX(pcre2_get_startchar_) @@ -848,6 +860,7 @@ pcre2_compile are called by application code. */ #define pcre2_jit_stack_create PCRE2_SUFFIX(pcre2_jit_stack_create_) #define pcre2_jit_stack_free PCRE2_SUFFIX(pcre2_jit_stack_free_) #define pcre2_maketables PCRE2_SUFFIX(pcre2_maketables_) +#define pcre2_maketables_free PCRE2_SUFFIX(pcre2_maketables_free_) #define pcre2_match PCRE2_SUFFIX(pcre2_match_) #define pcre2_match_context_copy PCRE2_SUFFIX(pcre2_match_context_copy_) #define pcre2_match_context_create PCRE2_SUFFIX(pcre2_match_context_create_) diff --git a/dist2/src/pcre2_auto_possess.c b/dist2/src/pcre2_auto_possess.c index 6d7b7c4a..c64cf856 100644 --- a/dist2/src/pcre2_auto_possess.c +++ b/dist2/src/pcre2_auto_possess.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -292,6 +292,7 @@ possessification, and if so, fills a list with its properties. Arguments: code points to start of expression utf TRUE if in UTF mode + ucp TRUE if in UCP mode fcc points to the case-flipping table list points to output list list[0] will be filled with the opcode @@ -304,7 +305,7 @@ Returns: points to the start of the next opcode if *code is accepted */ static PCRE2_SPTR -get_chr_property_list(PCRE2_SPTR code, BOOL utf, const uint8_t *fcc, +get_chr_property_list(PCRE2_SPTR code, BOOL utf, BOOL ucp, const uint8_t *fcc, uint32_t *list) { PCRE2_UCHAR c = *code; @@ -316,7 +317,8 @@ uint32_t chr; uint32_t *clist_dest; const uint32_t *clist_src; #else -(void)utf; /* Suppress "unused parameter" compiler warning */ +(void)utf; /* Suppress "unused parameter" compiler warnings */ +(void)ucp; #endif list[0] = c; @@ -396,7 +398,7 @@ switch(c) list[2] = chr; #ifdef SUPPORT_UNICODE - if (chr < 128 || (chr < 256 && !utf)) + if (chr < 128 || (chr < 256 && !utf && !ucp)) list[3] = fcc[chr]; else list[3] = UCD_OTHERCASE(chr); @@ -503,6 +505,7 @@ which case the base cannot be possessified. Arguments: code points to the byte code utf TRUE in UTF mode + ucp TRUE in UCP mode cb compile data block base_list the data list of the base opcode base_end the end of the base opcode @@ -512,7 +515,7 @@ Returns: TRUE if the auto-possessification is possible */ static BOOL -compare_opcodes(PCRE2_SPTR code, BOOL utf, const compile_block *cb, +compare_opcodes(PCRE2_SPTR code, BOOL utf, BOOL ucp, const compile_block *cb, const uint32_t *base_list, PCRE2_SPTR base_end, int *rec_limit) { PCRE2_UCHAR c; @@ -624,6 +627,13 @@ for(;;) case OP_ASSERTBACK_NOT: case OP_ONCE: return !entered_a_group; + + /* Non-atomic assertions - don't possessify last iterator. This needs + more thought. */ + + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: + return FALSE; } /* Skip over the bracket and inspect what comes next. */ @@ -644,7 +654,7 @@ for(;;) while (*next_code == OP_ALT) { - if (!compare_opcodes(code, utf, cb, base_list, base_end, rec_limit)) + if (!compare_opcodes(code, utf, ucp, cb, base_list, base_end, rec_limit)) return FALSE; code = next_code + 1 + LINK_SIZE; next_code += GET(next_code, 1); @@ -665,7 +675,8 @@ for(;;) /* The bracket content will be checked by the OP_BRA/OP_CBRA case above. */ next_code += 1 + LINK_SIZE; - if (!compare_opcodes(next_code, utf, cb, base_list, base_end, rec_limit)) + if (!compare_opcodes(next_code, utf, ucp, cb, base_list, base_end, + rec_limit)) return FALSE; code += PRIV(OP_lengths)[c]; @@ -681,7 +692,7 @@ for(;;) /* We now have the next appropriate opcode to compare with the base. Check for a supported opcode, and load its properties. */ - code = get_chr_property_list(code, utf, cb->fcc, list); + code = get_chr_property_list(code, utf, ucp, cb->fcc, list); if (code == NULL) return FALSE; /* Unsupported */ /* If either opcode is a small character list, set pointers for comparing @@ -1093,7 +1104,6 @@ leaving the remainder of the pattern unpossessified. Arguments: code points to start of the byte code - utf TRUE in UTF mode cb compile data block Returns: 0 for success @@ -1101,13 +1111,15 @@ Returns: 0 for success */ int -PRIV(auto_possessify)(PCRE2_UCHAR *code, BOOL utf, const compile_block *cb) +PRIV(auto_possessify)(PCRE2_UCHAR *code, const compile_block *cb) { PCRE2_UCHAR c; PCRE2_SPTR end; PCRE2_UCHAR *repeat_opcode; uint32_t list[8]; int rec_limit = 1000; /* Was 10,000 but clang+ASAN uses a lot of stack. */ +BOOL utf = (cb->external_options & PCRE2_UTF) != 0; +BOOL ucp = (cb->external_options & PCRE2_UCP) != 0; for (;;) { @@ -1119,10 +1131,11 @@ for (;;) { c -= get_repeat_base(c) - OP_STAR; end = (c <= OP_MINUPTO) ? - get_chr_property_list(code, utf, cb->fcc, list) : NULL; + get_chr_property_list(code, utf, ucp, cb->fcc, list) : NULL; list[1] = c == OP_STAR || c == OP_PLUS || c == OP_QUERY || c == OP_UPTO; - if (end != NULL && compare_opcodes(end, utf, cb, list, end, &rec_limit)) + if (end != NULL && compare_opcodes(end, utf, ucp, cb, list, end, + &rec_limit)) { switch(c) { @@ -1174,11 +1187,11 @@ for (;;) if (c >= OP_CRSTAR && c <= OP_CRMINRANGE) { /* end must not be NULL. */ - end = get_chr_property_list(code, utf, cb->fcc, list); + end = get_chr_property_list(code, utf, ucp, cb->fcc, list); list[1] = (c & 1) == 0; - if (compare_opcodes(end, utf, cb, list, end, &rec_limit)) + if (compare_opcodes(end, utf, ucp, cb, list, end, &rec_limit)) { switch (c) { diff --git a/dist2/src/pcre2_chartables.c b/dist2/src/pcre2_chartables.c index 0e07edb4..861914d1 100644 --- a/dist2/src/pcre2_chartables.c +++ b/dist2/src/pcre2_chartables.c @@ -2,17 +2,21 @@ * Perl-Compatible Regular Expressions * *************************************************/ -/* This file was automatically written by the dftables auxiliary +/* This file was automatically written by the pcre2_dftables auxiliary program. It contains character tables that are used when no external tables are passed to PCRE2 by the application that calls it. The tables are used only for characters whose code values are less than 256. */ -/*The dftables program (which is distributed with PCRE2) can be used to -build alternative versions of this file. This is necessary if you are +/* This set of tables was written in the C locale. */ + +/* The pcre2_ftables program (which is distributed with PCRE2) can be used +to build alternative versions of this file. This is necessary if you are running in an EBCDIC environment, or if you want to default to a different -encoding, for example ISO-8859-1. When dftables is run, it creates these -tables in the current locale. This happens automatically if PCRE2 is -configured with --enable-rebuild-chartables. */ +encoding, for example ISO-8859-1. When pcre2_dftables is run, it creates +these tables in the "C" locale by default. This happens automatically if +PCRE2 is configured with --enable-rebuild-chartables. However, you can run +pcre2_dftables manually with the -L option to build tables using the LC_ALL +locale. */ /* The following #include is present because without it gcc 4.x may remove the array definition from the final binary if PCRE2 is built into a static @@ -102,54 +106,54 @@ const uint8_t PRIV(default_tables)[] = { /* This table contains bit maps for various character classes. Each map is 32 bytes long and the bits run from the least significant end of each byte. The classes that have their own maps are: space, xdigit, digit, upper, lower, word, -graph print, punct, and cntrl. Other classes are built from combinations. */ +graph, print, punct, and cntrl. Other classes are built from combinations. */ - 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, /* space */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* xdigit */ 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* digit */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* upper */ 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* lower */ 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* word */ 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, /* graph */ 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, + 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, /* print */ 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, + 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, /* punct */ 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, + 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, /* cntrl */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, diff --git a/dist2/src/pcre2_chartables.c.dist b/dist2/src/pcre2_chartables.c.dist index 0e07edb4..861914d1 100644 --- a/dist2/src/pcre2_chartables.c.dist +++ b/dist2/src/pcre2_chartables.c.dist @@ -2,17 +2,21 @@ * Perl-Compatible Regular Expressions * *************************************************/ -/* This file was automatically written by the dftables auxiliary +/* This file was automatically written by the pcre2_dftables auxiliary program. It contains character tables that are used when no external tables are passed to PCRE2 by the application that calls it. The tables are used only for characters whose code values are less than 256. */ -/*The dftables program (which is distributed with PCRE2) can be used to -build alternative versions of this file. This is necessary if you are +/* This set of tables was written in the C locale. */ + +/* The pcre2_ftables program (which is distributed with PCRE2) can be used +to build alternative versions of this file. This is necessary if you are running in an EBCDIC environment, or if you want to default to a different -encoding, for example ISO-8859-1. When dftables is run, it creates these -tables in the current locale. This happens automatically if PCRE2 is -configured with --enable-rebuild-chartables. */ +encoding, for example ISO-8859-1. When pcre2_dftables is run, it creates +these tables in the "C" locale by default. This happens automatically if +PCRE2 is configured with --enable-rebuild-chartables. However, you can run +pcre2_dftables manually with the -L option to build tables using the LC_ALL +locale. */ /* The following #include is present because without it gcc 4.x may remove the array definition from the final binary if PCRE2 is built into a static @@ -102,54 +106,54 @@ const uint8_t PRIV(default_tables)[] = { /* This table contains bit maps for various character classes. Each map is 32 bytes long and the bits run from the least significant end of each byte. The classes that have their own maps are: space, xdigit, digit, upper, lower, word, -graph print, punct, and cntrl. Other classes are built from combinations. */ +graph, print, punct, and cntrl. Other classes are built from combinations. */ - 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, /* space */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* xdigit */ 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* digit */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* upper */ 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* lower */ 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, /* word */ 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, /* graph */ 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, + 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, /* print */ 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, + 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, /* punct */ 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, + 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, /* cntrl */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, diff --git a/dist2/src/pcre2_compile.c b/dist2/src/pcre2_compile.c index 068735ae..e811f12f 100644 --- a/dist2/src/pcre2_compile.c +++ b/dist2/src/pcre2_compile.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -135,6 +135,9 @@ static BOOL set_lookbehind_lengths(uint32_t **, int *, int *, parsed_recurse_check *, compile_block *); +static int + check_lookbehinds(uint32_t *, uint32_t **, parsed_recurse_check *, + compile_block *); /************************************************* @@ -250,36 +253,41 @@ is present where expected in a conditional group. */ #define META_LOOKBEHIND 0x80250000u /* (?<= */ #define META_LOOKBEHINDNOT 0x80260000u /* (?<! */ +/* These cannot be conditions */ + +#define META_LOOKAHEAD_NA 0x80270000u /* (*napla: */ +#define META_LOOKBEHIND_NA 0x80280000u /* (*naplb: */ + /* These must be kept in this order, with consecutive values, and the _ARG versions of COMMIT, PRUNE, SKIP, and THEN immediately after their non-argument versions. */ -#define META_MARK 0x80270000u /* (*MARK) */ -#define META_ACCEPT 0x80280000u /* (*ACCEPT) */ -#define META_FAIL 0x80290000u /* (*FAIL) */ -#define META_COMMIT 0x802a0000u /* These */ -#define META_COMMIT_ARG 0x802b0000u /* pairs */ -#define META_PRUNE 0x802c0000u /* must */ -#define META_PRUNE_ARG 0x802d0000u /* be */ -#define META_SKIP 0x802e0000u /* kept */ -#define META_SKIP_ARG 0x802f0000u /* in */ -#define META_THEN 0x80300000u /* this */ -#define META_THEN_ARG 0x80310000u /* order */ +#define META_MARK 0x80290000u /* (*MARK) */ +#define META_ACCEPT 0x802a0000u /* (*ACCEPT) */ +#define META_FAIL 0x802b0000u /* (*FAIL) */ +#define META_COMMIT 0x802c0000u /* These */ +#define META_COMMIT_ARG 0x802d0000u /* pairs */ +#define META_PRUNE 0x802e0000u /* must */ +#define META_PRUNE_ARG 0x802f0000u /* be */ +#define META_SKIP 0x80300000u /* kept */ +#define META_SKIP_ARG 0x80310000u /* in */ +#define META_THEN 0x80320000u /* this */ +#define META_THEN_ARG 0x80330000u /* order */ /* These must be kept in groups of adjacent 3 values, and all together. */ -#define META_ASTERISK 0x80320000u /* * */ -#define META_ASTERISK_PLUS 0x80330000u /* *+ */ -#define META_ASTERISK_QUERY 0x80340000u /* *? */ -#define META_PLUS 0x80350000u /* + */ -#define META_PLUS_PLUS 0x80360000u /* ++ */ -#define META_PLUS_QUERY 0x80370000u /* +? */ -#define META_QUERY 0x80380000u /* ? */ -#define META_QUERY_PLUS 0x80390000u /* ?+ */ -#define META_QUERY_QUERY 0x803a0000u /* ?? */ -#define META_MINMAX 0x803b0000u /* {n,m} repeat */ -#define META_MINMAX_PLUS 0x803c0000u /* {n,m}+ repeat */ -#define META_MINMAX_QUERY 0x803d0000u /* {n,m}? repeat */ +#define META_ASTERISK 0x80340000u /* * */ +#define META_ASTERISK_PLUS 0x80350000u /* *+ */ +#define META_ASTERISK_QUERY 0x80360000u /* *? */ +#define META_PLUS 0x80370000u /* + */ +#define META_PLUS_PLUS 0x80380000u /* ++ */ +#define META_PLUS_QUERY 0x80390000u /* +? */ +#define META_QUERY 0x803a0000u /* ? */ +#define META_QUERY_PLUS 0x803b0000u /* ?+ */ +#define META_QUERY_QUERY 0x803c0000u /* ?? */ +#define META_MINMAX 0x803d0000u /* {n,m} repeat */ +#define META_MINMAX_PLUS 0x803e0000u /* {n,m}+ repeat */ +#define META_MINMAX_QUERY 0x803f0000u /* {n,m}? repeat */ #define META_FIRST_QUANTIFIER META_ASTERISK #define META_LAST_QUANTIFIER META_MINMAX_QUERY @@ -335,6 +343,8 @@ static unsigned char meta_extra_lengths[] = { 0, /* META_LOOKAHEADNOT */ SIZEOFFSET, /* META_LOOKBEHIND */ SIZEOFFSET, /* META_LOOKBEHINDNOT */ + 0, /* META_LOOKAHEAD_NA */ + SIZEOFFSET, /* META_LOOKBEHIND_NA */ 1, /* META_MARK - plus the string length */ 0, /* META_ACCEPT */ 0, /* META_FAIL */ @@ -634,10 +644,14 @@ typedef struct alasitem { static const char alasnames[] = STRING_pla0 STRING_plb0 + STRING_napla0 + STRING_naplb0 STRING_nla0 STRING_nlb0 STRING_positive_lookahead0 STRING_positive_lookbehind0 + STRING_non_atomic_positive_lookahead0 + STRING_non_atomic_positive_lookbehind0 STRING_negative_lookahead0 STRING_negative_lookbehind0 STRING_atomic0 @@ -649,10 +663,14 @@ static const char alasnames[] = static const alasitem alasmeta[] = { { 3, META_LOOKAHEAD }, { 3, META_LOOKBEHIND }, + { 5, META_LOOKAHEAD_NA }, + { 5, META_LOOKBEHIND_NA }, { 3, META_LOOKAHEADNOT }, { 3, META_LOOKBEHINDNOT }, { 18, META_LOOKAHEAD }, { 19, META_LOOKBEHIND }, + { 29, META_LOOKAHEAD_NA }, + { 30, META_LOOKBEHIND_NA }, { 18, META_LOOKAHEADNOT }, { 19, META_LOOKBEHINDNOT }, { 6, META_ATOMIC }, @@ -746,8 +764,8 @@ are allowed. */ #define PUBLIC_LITERAL_COMPILE_OPTIONS \ (PCRE2_ANCHORED|PCRE2_AUTO_CALLOUT|PCRE2_CASELESS|PCRE2_ENDANCHORED| \ - PCRE2_FIRSTLINE|PCRE2_LITERAL|PCRE2_NO_START_OPTIMIZE| \ - PCRE2_NO_UTF_CHECK|PCRE2_USE_OFFSET_LIMIT|PCRE2_UTF) + PCRE2_FIRSTLINE|PCRE2_LITERAL|PCRE2_MATCH_INVALID_UTF| \ + PCRE2_NO_START_OPTIMIZE|PCRE2_NO_UTF_CHECK|PCRE2_USE_OFFSET_LIMIT|PCRE2_UTF) #define PUBLIC_COMPILE_OPTIONS \ (PUBLIC_LITERAL_COMPILE_OPTIONS| \ @@ -781,7 +799,7 @@ enum { ERR0 = COMPILE_ERROR_BASE, ERR61, ERR62, ERR63, ERR64, ERR65, ERR66, ERR67, ERR68, ERR69, ERR70, ERR71, ERR72, ERR73, ERR74, ERR75, ERR76, ERR77, ERR78, ERR79, ERR80, ERR81, ERR82, ERR83, ERR84, ERR85, ERR86, ERR87, ERR88, ERR89, ERR90, - ERR91, ERR92, ERR93, ERR94, ERR95, ERR96 }; + ERR91, ERR92, ERR93, ERR94, ERR95, ERR96, ERR97, ERR98 }; /* This is a table of start-of-pattern options such as (*UTF) and settings such as (*LIMIT_MATCH=nnnn) and (*CRLF). For completeness and backward @@ -1012,6 +1030,7 @@ for (;;) case META_NOCAPTURE: fprintf(stderr, "META (?:"); break; case META_LOOKAHEAD: fprintf(stderr, "META (?="); break; case META_LOOKAHEADNOT: fprintf(stderr, "META (?!"); break; + case META_LOOKAHEAD_NA: fprintf(stderr, "META (*napla:"); break; case META_SCRIPT_RUN: fprintf(stderr, "META (*sr:"); break; case META_KET: fprintf(stderr, "META )"); break; case META_ALT: fprintf(stderr, "META | %d", meta_arg); break; @@ -1043,6 +1062,12 @@ for (;;) fprintf(stderr, "%zd", offset); break; + case META_LOOKBEHIND_NA: + fprintf(stderr, "META (*naplb: %d offset=", meta_arg); + GETOFFSET(offset, pptr); + fprintf(stderr, "%zd", offset); + break; + case META_LOOKBEHINDNOT: fprintf(stderr, "META (?<! %d offset=", meta_arg); GETOFFSET(offset, pptr); @@ -1177,7 +1202,7 @@ in the decoded tables. */ if ((code->flags & PCRE2_DEREF_TABLES) != 0) { - ref_count = (PCRE2_SIZE *)(code->tables + tables_length); + ref_count = (PCRE2_SIZE *)(code->tables + TABLES_LENGTH); (*ref_count)++; } @@ -1207,15 +1232,15 @@ if (newcode == NULL) return NULL; memcpy(newcode, code, code->blocksize); newcode->executable_jit = NULL; -newtables = code->memctl.malloc(tables_length + sizeof(PCRE2_SIZE), +newtables = code->memctl.malloc(TABLES_LENGTH + sizeof(PCRE2_SIZE), code->memctl.memory_data); if (newtables == NULL) { code->memctl.free((void *)newcode, code->memctl.memory_data); return NULL; } -memcpy(newtables, code->tables, tables_length); -ref_count = (PCRE2_SIZE *)(newtables + tables_length); +memcpy(newtables, code->tables, TABLES_LENGTH); +ref_count = (PCRE2_SIZE *)(newtables + TABLES_LENGTH); *ref_count = 1; newcode->tables = newtables; @@ -1245,7 +1270,7 @@ if (code != NULL) be freed when there are no more references to them. The *ref_count should always be > 0. */ - ref_count = (PCRE2_SIZE *)(code->tables + tables_length); + ref_count = (PCRE2_SIZE *)(code->tables + TABLES_LENGTH); if (*ref_count > 0) { (*ref_count)--; @@ -1419,9 +1444,6 @@ the result is "not a repeat quantifier". */ EXIT: if (yield || *errorcodeptr != 0) *ptrptr = p; return yield; - - - } @@ -2322,7 +2344,7 @@ if (ptr > *nameptr + MAX_NAME_SIZE) *errorcodeptr = ERR48; goto FAILED; } -*namelenptr = ptr - *nameptr; +*namelenptr = (uint32_t)(ptr - *nameptr); /* Subpattern names must not be empty, and their terminator is checked here. (What follows a verb or alpha assertion name is checked separately.) */ @@ -2450,8 +2472,9 @@ must be last. */ enum { RANGE_NO, RANGE_STARTED, RANGE_OK_ESCAPED, RANGE_OK_LITERAL }; -/* Only in 32-bit mode can there be literals > META_END. A macros encapsulates -the storing of literal values in the parsed pattern. */ +/* Only in 32-bit mode can there be literals > META_END. A macro encapsulates +the storing of literal values in the main parsed pattern, where they can always +be quantified. */ #if PCRE2_CODE_UNIT_WIDTH == 32 #define PARSED_LITERAL(c, p) \ @@ -2474,6 +2497,7 @@ uint32_t delimiter; uint32_t namelen; uint32_t class_range_state; uint32_t *verblengthptr = NULL; /* Value avoids compiler warning */ +uint32_t *verbstartptr = NULL; uint32_t *previous_callout = NULL; uint32_t *parsed_pattern = cb->parsed_pattern; uint32_t *parsed_pattern_end = cb->parsed_pattern_end; @@ -2600,10 +2624,20 @@ while (ptr < ptrend) errorcode = ERR28; goto FAILED; } - if (!inverbname && after_manual_callout-- <= 0) - parsed_pattern = manage_callouts(thisptr, &previous_callout, - auto_callout, parsed_pattern, cb); - PARSED_LITERAL(c, parsed_pattern); + if (inverbname) + { /* Don't use PARSED_LITERAL() because it */ +#if PCRE2_CODE_UNIT_WIDTH == 32 /* sets okquantifier. */ + if (c >= META_END) *parsed_pattern++ = META_BIGVALUE; +#endif + *parsed_pattern++ = c; + } + else + { + if (after_manual_callout-- <= 0) + parsed_pattern = manage_callouts(thisptr, &previous_callout, + auto_callout, parsed_pattern, cb); + PARSED_LITERAL(c, parsed_pattern); + } meta_quantifier = 0; } continue; /* Next character */ @@ -2640,13 +2674,15 @@ while (ptr < ptrend) switch(c) { - default: - PARSED_LITERAL(c, parsed_pattern); + default: /* Don't use PARSED_LITERAL() because it */ +#if PCRE2_CODE_UNIT_WIDTH == 32 /* sets okquantifier. */ + if (c >= META_END) *parsed_pattern++ = META_BIGVALUE; +#endif + *parsed_pattern++ = c; break; case CHAR_RIGHT_PARENTHESIS: inverbname = FALSE; - okquantifier = FALSE; /* Was probably set by literals */ /* This is the length in characters */ verbnamelength = (PCRE2_SIZE)(parsed_pattern - verblengthptr - 1); /* But the limit on the length is in code units */ @@ -2680,8 +2716,11 @@ while (ptr < ptrend) switch(escape) { - case 0: - PARSED_LITERAL(c, parsed_pattern); + case 0: /* Don't use PARSED_LITERAL() because it */ +#if PCRE2_CODE_UNIT_WIDTH == 32 /* sets okquantifier. */ + if (c >= META_END) *parsed_pattern++ = META_BIGVALUE; +#endif + *parsed_pattern++ = c; break; case ESC_Q: @@ -3135,6 +3174,21 @@ while (ptr < ptrend) goto FAILED_BACK; } + /* Most (*VERB)s are not allowed to be quantified, but an ungreedy + quantifier can be useful for (*ACCEPT) - meaning "succeed on backtrack", a + sort of negated (*COMMIT). We therefore allow (*ACCEPT) to be quantified by + wrapping it in non-capturing brackets, but we have to allow for a preceding + (*MARK) for when (*ACCEPT) has an argument. */ + + if (parsed_pattern[-1] == META_ACCEPT) + { + uint32_t *p; + for (p = parsed_pattern - 1; p >= verbstartptr; p--) p[1] = p[0]; + *verbstartptr = META_NOCAPTURE; + parsed_pattern[1] = META_KET; + parsed_pattern += 2; + } + /* Now we can put the quantifier into the parsed pattern vector. At this stage, we have only the basic quantifier. The check for a following + or ? modifier happens at the top of the loop, after any intervening comments @@ -3581,6 +3635,8 @@ while (ptr < ptrend) if (c == CHAR_RIGHT_SQUARE_BRACKET && !inescq) break; } /* End of class-processing loop */ + /* -] at the end of a class is a literal '-' */ + if (class_range_state == RANGE_STARTED) { parsed_pattern[-1] = CHAR_MINUS; @@ -3597,7 +3653,7 @@ while (ptr < ptrend) if (ptr >= ptrend) goto UNCLOSED_PARENTHESIS; /* If ( is not followed by ? it is either a capture or a special verb or an - alpha assertion. */ + alpha assertion or a positive non-atomic lookahead. */ if (*ptr != CHAR_QUESTION_MARK) { @@ -3611,6 +3667,11 @@ while (ptr < ptrend) nest_depth++; if ((options & PCRE2_NO_AUTO_CAPTURE) == 0) { + if (cb->bracount >= MAX_GROUP_NUMBER) + { + errorcode = ERR97; + goto FAILED; + } cb->bracount++; *parsed_pattern++ = META_CAPTURE | cb->bracount; } @@ -3624,10 +3685,10 @@ while (ptr < ptrend) break; /* Handle "alpha assertions" such as (*pla:...). Most of these are - synonyms for the historical symbolic assertions, but the script run ones - are new. They are distinguished by starting with a lower case letter. - Checking both ends of the alphabet makes this work in all character - codes. */ + synonyms for the historical symbolic assertions, but the script run and + non-atomic lookaround ones are new. They are distinguished by starting + with a lower case letter. Checking both ends of the alphabet makes this + work in all character codes. */ else if (CHMAX_255(c) && (cb->ctypes[c] & ctype_lcletter) != 0) { @@ -3658,19 +3719,20 @@ while (ptr < ptrend) goto FAILED; } - /* Check for expecting an assertion condition. If so, only lookaround - assertions are valid. */ + /* Check for expecting an assertion condition. If so, only atomic + lookaround assertions are valid. */ meta = alasmeta[i].meta; if (prev_expect_cond_assert > 0 && (meta < META_LOOKAHEAD || meta > META_LOOKBEHINDNOT)) { - errorcode = ERR28; /* Assertion expected */ + errorcode = (meta == META_LOOKAHEAD_NA || meta == META_LOOKBEHIND_NA)? + ERR98 : ERR28; /* (Atomic) assertion expected */ goto FAILED; } - /* The lookaround alphabetic synonyms can be almost entirely handled by - jumping to the code that handles the traditional symbolic forms. */ + /* The lookaround alphabetic synonyms can mostly be handled by jumping + to the code that handles the traditional symbolic forms. */ switch(meta) { @@ -3684,11 +3746,15 @@ while (ptr < ptrend) case META_LOOKAHEAD: goto POSITIVE_LOOK_AHEAD; + case META_LOOKAHEAD_NA: + goto POSITIVE_NONATOMIC_LOOK_AHEAD; + case META_LOOKAHEADNOT: goto NEGATIVE_LOOK_AHEAD; case META_LOOKBEHIND: case META_LOOKBEHINDNOT: + case META_LOOKBEHIND_NA: *parsed_pattern++ = meta; ptr--; goto POST_LOOKBEHIND; @@ -3770,6 +3836,12 @@ while (ptr < ptrend) goto FAILED; } + /* Remember where this verb, possibly with a preceding (*MARK), starts, + for handling quantified (*ACCEPT). */ + + verbstartptr = parsed_pattern; + okquantifier = (verbs[i].meta == META_ACCEPT); + /* It appears that Perl allows any characters whatsoever, other than a closing parenthesis, to appear in arguments ("names"), so we no longer insist on letters, digits, and underscores. Perl does not, however, do @@ -4259,6 +4331,7 @@ while (ptr < ptrend) { if (++ptr >= ptrend || !IS_DIGIT(*ptr)) goto BAD_VERSION_CONDITION; minor = (*ptr++ - CHAR_0) * 10; + if (ptr >= ptrend) goto BAD_VERSION_CONDITION; if (IS_DIGIT(*ptr)) minor += *ptr++ - CHAR_0; if (ptr >= ptrend || *ptr != CHAR_RIGHT_PARENTHESIS) goto BAD_VERSION_CONDITION; @@ -4364,6 +4437,12 @@ while (ptr < ptrend) ptr++; goto POST_ASSERTION; + case CHAR_ASTERISK: + POSITIVE_NONATOMIC_LOOK_AHEAD: /* Come from (?* */ + *parsed_pattern++ = META_LOOKAHEAD_NA; + ptr++; + goto POST_ASSERTION; + case CHAR_EXCLAMATION_MARK: NEGATIVE_LOOK_AHEAD: /* Come from (*nla: */ *parsed_pattern++ = META_LOOKAHEADNOT; @@ -4373,20 +4452,23 @@ while (ptr < ptrend) /* ---- Lookbehind assertions ---- */ - /* (?< followed by = or ! is a lookbehind assertion. Otherwise (?< is the - start of the name of a capturing group. */ + /* (?< followed by = or ! or * is a lookbehind assertion. Otherwise (?< + is the start of the name of a capturing group. */ case CHAR_LESS_THAN_SIGN: if (ptrend - ptr <= 1 || - (ptr[1] != CHAR_EQUALS_SIGN && ptr[1] != CHAR_EXCLAMATION_MARK)) + (ptr[1] != CHAR_EQUALS_SIGN && + ptr[1] != CHAR_EXCLAMATION_MARK && + ptr[1] != CHAR_ASTERISK)) { terminator = CHAR_GREATER_THAN_SIGN; goto DEFINE_NAME; } *parsed_pattern++ = (ptr[1] == CHAR_EQUALS_SIGN)? - META_LOOKBEHIND : META_LOOKBEHINDNOT; + META_LOOKBEHIND : (ptr[1] == CHAR_EXCLAMATION_MARK)? + META_LOOKBEHINDNOT : META_LOOKBEHIND_NA; - POST_LOOKBEHIND: /* Come from (*plb: and (*nlb: */ + POST_LOOKBEHIND: /* Come from (*plb: (*naplb: and (*nlb: */ *has_lookbehind = TRUE; offset = (PCRE2_SIZE)(ptr - cb->start_pattern - 2); PUTOFFSET(offset, parsed_pattern); @@ -4435,6 +4517,11 @@ while (ptr < ptrend) /* We have a name for this capturing group. It is also assigned a number, which is its primary means of identification. */ + if (cb->bracount >= MAX_GROUP_NUMBER) + { + errorcode = ERR97; + goto FAILED; + } cb->bracount++; *parsed_pattern++ = META_CAPTURE | cb->bracount; nest_depth++; @@ -4554,8 +4641,6 @@ while (ptr < ptrend) *parsed_pattern++ = META_KET; } - - if (top_nest == (nest_save *)(cb->start_workspace)) top_nest = NULL; else top_nest--; } @@ -4661,6 +4746,7 @@ for (;;) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERTBACK_NA: if (!skipassert) return code; do code += GET(code, 1); while (*code == OP_ALT); code += PRIV(OP_lengths)[*code]; @@ -4819,7 +4905,7 @@ range. */ if ((options & PCRE2_CASELESS) != 0) { #ifdef SUPPORT_UNICODE - if ((options & PCRE2_UTF) != 0) + if ((options & (PCRE2_UTF|PCRE2_UCP)) != 0) { int rc; uint32_t oc, od; @@ -5221,8 +5307,10 @@ PCRE2_UCHAR *tempcode; PCRE2_UCHAR *previous = NULL; PCRE2_UCHAR op_previous; BOOL groupsetfirstcu = FALSE; +BOOL had_accept = FALSE; BOOL matched_char = FALSE; BOOL previous_matched_char = FALSE; +BOOL reset_caseful = FALSE; const uint8_t *cbits = cb->cbits; uint8_t classbits[32]; @@ -5232,7 +5320,8 @@ dynamically as we process the pattern. */ #ifdef SUPPORT_UNICODE BOOL utf = (options & PCRE2_UTF) != 0; -#else /* No UTF support */ +BOOL ucp = (options & PCRE2_UCP) != 0; +#else /* No Unicode support */ BOOL utf = FALSE; #endif @@ -5355,7 +5444,7 @@ for (;; pptr++) if (meta < META_ASTERISK || meta > META_MINMAX_QUERY) { previous = code; - if (matched_char) okreturn = 1; + if (matched_char && !had_accept) okreturn = 1; } previous_matched_char = matched_char; @@ -5477,12 +5566,12 @@ for (;; pptr++) zerofirstcu = firstcu; zerofirstcuflags = firstcuflags; - /* For caseless UTF mode, check whether this character has more than - one other case. If so, generate a special OP_NOTPROP item instead of + /* For caseless UTF or UCP mode, check whether this character has more + than one other case. If so, generate a special OP_NOTPROP item instead of OP_NOTI. */ #ifdef SUPPORT_UNICODE - if (utf && (options & PCRE2_CASELESS) != 0 && + if ((utf||ucp) && (options & PCRE2_CASELESS) != 0 && (d = UCD_CASESET(c)) != 0) { *code++ = OP_NOTPROP; @@ -5499,7 +5588,45 @@ for (;; pptr++) } /* End of 1-char optimization */ /* Handle character classes that contain more than just one literal - character. */ + character. If there are exactly two characters in a positive class, see if + they are case partners. This can be optimized to generate a caseless single + character match (which also sets first/required code units if relevant). */ + + if (meta == META_CLASS && pptr[1] < META_END && pptr[2] < META_END && + pptr[3] == META_CLASS_END) + { + uint32_t c = pptr[1]; + +#ifdef SUPPORT_UNICODE + if (UCD_CASESET(c) == 0) +#endif + { + uint32_t d; + +#ifdef SUPPORT_UNICODE + if ((utf || ucp) && c > 127) d = UCD_OTHERCASE(c); else +#endif + { +#if PCRE2_CODE_UNIT_WIDTH != 8 + if (c > 255) d = c; else +#endif + d = TABLE_GET(c, cb->fcc, c); + } + + if (c != d && pptr[2] == d) + { + pptr += 3; /* Move on to class end */ + meta = c; + if ((options & PCRE2_CASELESS) == 0) + { + reset_caseful = TRUE; + options |= PCRE2_CASELESS; + req_caseopt = REQ_CASELESS; + } + goto CLASS_CASELESS_CHAR; + } + } + } /* If a non-extended class contains a negative special such as \S, we need to flip the negation flag at the end, so that support for characters > 255 @@ -5994,7 +6121,7 @@ for (;; pptr++) workspace overflow. Do not set firstcu after *ACCEPT. */ case META_ACCEPT: - cb->had_accept = TRUE; + cb->had_accept = had_accept = TRUE; for (oc = cb->open_caps; oc != NULL && oc->assert_depth >= cb->assert_depth; oc = oc->next) @@ -6252,6 +6379,11 @@ for (;; pptr++) cb->assert_depth += 1; goto GROUP_PROCESS; + case META_LOOKAHEAD_NA: + bravalue = OP_ASSERT_NA; + cb->assert_depth += 1; + goto GROUP_PROCESS; + /* Optimize (?!) to (*FAIL) unless it is quantified - which is a weird thing to do, but Perl allows all assertions to be quantified, and when they contain capturing parentheses there may be a potential use for @@ -6283,6 +6415,11 @@ for (;; pptr++) cb->assert_depth += 1; goto GROUP_PROCESS; + case META_LOOKBEHIND_NA: + bravalue = OP_ASSERTBACK_NA; + cb->assert_depth += 1; + goto GROUP_PROCESS; + case META_ATOMIC: bravalue = OP_ONCE; goto GROUP_PROCESS_NOTE_EMPTY; @@ -6341,7 +6478,7 @@ for (;; pptr++) /* If we've just compiled an assertion, pop the assert depth. */ - if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT) + if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NA) cb->assert_depth -= 1; /* At the end of compiling, code is still pointing to the start of the @@ -6491,8 +6628,8 @@ for (;; pptr++) we must only take the reqcu when the group also set a firstcu. Otherwise, in that example, 'X' ends up set for both. */ - else if (bravalue == OP_ASSERT && subreqcuflags >= 0 && - subfirstcuflags >= 0) + else if ((bravalue == OP_ASSERT || bravalue == OP_ASSERT_NA) && + subreqcuflags >= 0 && subfirstcuflags >= 0) { reqcu = subreqcu; reqcuflags = subreqcuflags; @@ -6541,23 +6678,11 @@ for (;; pptr++) } /* For a back reference, update the back reference map and the - maximum back reference. Then, for each group, we must check to - see if it is recursive, that is, it is inside the group that it - references. A flag is set so that the group can be made atomic. - */ + maximum back reference. */ cb->backref_map |= (groupnumber < 32)? (1u << groupnumber) : 1; if (groupnumber > cb->top_backref) cb->top_backref = groupnumber; - - for (oc = cb->open_caps; oc != NULL; oc = oc->next) - { - if (oc->number == groupnumber) - { - oc->flag = TRUE; - break; - } - } } } @@ -6713,10 +6838,6 @@ for (;; pptr++) reqvary = (repeat_min == repeat_max)? 0 : REQ_VARY; op_type = 0; - /* If the repeat is {1} we can ignore it. */ - - if (repeat_max == 1 && repeat_min == 1) goto END_REPEAT; - /* Adjust first and required code units for a zero repeat. */ if (repeat_min == 0) @@ -6759,7 +6880,10 @@ for (;; pptr++) tempcode = previous; op_previous = *previous; - /* Now handle repetition for the different types of item. */ + /* Now handle repetition for the different types of item. If the repeat + minimum and the repeat maximum are both 1, we can ignore the quantifier for + non-parenthesized items, as they have only one alternative. For anything in + parentheses, we must not ignore if {1} is possessive. */ switch (op_previous) { @@ -6773,6 +6897,7 @@ for (;; pptr++) case OP_CHARI: case OP_NOT: case OP_NOTI: + if (repeat_max == 1 && repeat_min == 1) goto END_REPEAT; op_type = chartypeoffset[op_previous - OP_CHAR]; /* Deal with UTF characters that take up more than one code unit. */ @@ -6819,6 +6944,7 @@ for (;; pptr++) code = previous; goto END_REPEAT; } + if (repeat_max == 1 && repeat_min == 1) goto END_REPEAT; if (repeat_min == 0 && repeat_max == REPEAT_UNLIMITED) *code++ = OP_CRSTAR + repeat_type; @@ -6853,6 +6979,8 @@ for (;; pptr++) repetition. */ case OP_RECURSE: + if (repeat_max == 1 && repeat_min == 1 && !possessive_quantifier) + goto END_REPEAT; /* Generate unwrapped repeats for a non-zero minimum, except when the minimum is 1 and the maximum unlimited, because that can be handled with @@ -6923,8 +7051,10 @@ for (;; pptr++) case OP_ASSERT: case OP_ASSERT_NOT: + case OP_ASSERT_NA: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRA: @@ -6935,6 +7065,9 @@ for (;; pptr++) PCRE2_UCHAR *bralink = NULL; PCRE2_UCHAR *brazeroptr = NULL; + if (repeat_max == 1 && repeat_min == 1 && !possessive_quantifier) + goto END_REPEAT; + /* Repeating a DEFINE group (or any group where the condition is always FALSE and there is only one branch) is pointless, but Perl allows the syntax, so we just ignore the repeat. */ @@ -6943,15 +7076,18 @@ for (;; pptr++) previous[GET(previous, 1)] != OP_ALT) goto END_REPEAT; - /* There is no sense in actually repeating assertions. The only - potential use of repetition is in cases when the assertion is optional. - Therefore, if the minimum is greater than zero, just ignore the repeat. - If the maximum is not zero or one, set it to 1. */ + /* Perl allows all assertions to be quantified, and when they contain + capturing parentheses and/or are optional there are potential uses for + this feature. PCRE2 used to force the maximum quantifier to 1 on the + invalid grounds that further repetition was never useful. This was + always a bit pointless, since an assertion could be wrapped with a + repeated group to achieve the effect. General repetition is now + permitted, but if the maximum is unlimited it is set to one more than + the minimum. */ if (op_previous < OP_ONCE) /* Assertion */ { - if (repeat_min > 0) goto END_REPEAT; - if (repeat_max > 1) repeat_max = 1; + if (repeat_max == REPEAT_UNLIMITED) repeat_max = repeat_min + 1; } /* The case of a zero minimum is special because of the need to stick @@ -7151,11 +7287,12 @@ for (;; pptr++) and SCRIPT_RUN groups at runtime, but in a different way.] Then, if the quantifier was possessive and the bracket is not a - conditional, we convert the BRA code to the POS form, and the KET code to - KETRPOS. (It turns out to be convenient at runtime to detect this kind of - subpattern at both the start and at the end.) The use of special opcodes - makes it possible to reduce greatly the stack usage in pcre2_match(). If - the group is preceded by OP_BRAZERO, convert this to OP_BRAPOSZERO. + conditional, we convert the BRA code to the POS form, and the KET code + to KETRPOS. (It turns out to be convenient at runtime to detect this + kind of subpattern at both the start and at the end.) The use of + special opcodes makes it possible to reduce greatly the stack usage in + pcre2_match(). If the group is preceded by OP_BRAZERO, convert this to + OP_BRAPOSZERO. Then, if the minimum number of matches is 1 or 0, cancel the possessive flag so that the default action below, of wrapping everything inside @@ -7256,6 +7393,8 @@ for (;; pptr++) int prop_type, prop_value; PCRE2_UCHAR *oldcode; + if (repeat_max == 1 && repeat_min == 1) goto END_REPEAT; + op_type = OP_TYPESTAR - OP_STAR; /* Use type opcodes */ mclength = 0; /* Not a character */ @@ -7541,19 +7680,6 @@ for (;; pptr++) cb->backref_map |= (meta_arg < 32)? (1u << meta_arg) : 1; if (meta_arg > cb->top_backref) cb->top_backref = meta_arg; - - /* Check to see if this back reference is recursive, that it, it - is inside the group that it references. A flag is set so that the - group can be made atomic. */ - - for (oc = cb->open_caps; oc != NULL; oc = oc->next) - { - if (oc->number == meta_arg) - { - oc->flag = TRUE; - break; - } - } break; @@ -7699,11 +7825,12 @@ for (;; pptr++) NORMAL_CHAR_SET: /* Character is already in meta */ matched_char = TRUE; - /* For caseless UTF mode, check whether this character has more than one - other case. If so, generate a special OP_PROP item instead of OP_CHARI. */ + /* For caseless UTF or UCP mode, check whether this character has more than + one other case. If so, generate a special OP_PROP item instead of OP_CHARI. + */ #ifdef SUPPORT_UNICODE - if (utf && (options & PCRE2_CASELESS) != 0) + if ((utf||ucp) && (options & PCRE2_CASELESS) != 0) { uint32_t caseset = UCD_CASESET(meta); if (caseset != 0) @@ -7718,9 +7845,15 @@ for (;; pptr++) } #endif - /* Caseful matches, or not one of the multicase characters. Get the - character's code units into mcbuffer, with the length in mclength. When not - in UTF mode, the length is always 1. */ + /* Caseful matches, or caseless and not one of the multicase characters. We + come here by goto in the case of a positive class that contains only + case-partners of a character with just two cases; matched_char has already + been set TRUE and options fudged if necessary. */ + + CLASS_CASELESS_CHAR: + + /* Get the character's code units into mcbuffer, with the length in + mclength. When not in UTF mode, the length is always 1. */ #ifdef SUPPORT_UNICODE if (utf) mclength = PRIV(ord2utf)(meta, mcbuffer); else @@ -7752,8 +7885,9 @@ for (;; pptr++) zeroreqcu = reqcu; zeroreqcuflags = reqcuflags; - /* If the character is more than one code unit long, we can set firstcu - only if it is not to be matched caselessly. */ + /* If the character is more than one code unit long, we can set a single + firstcu only if it is not to be matched caselessly. Multiple possible + starting code units may be picked up later in the studying code. */ if (mclength == 1 || req_caseopt == 0) { @@ -7783,7 +7917,17 @@ for (;; pptr++) reqcuflags = req_caseopt | cb->req_varyopt; } } - break; /* End default meta handling */ + + /* If caselessness was temporarily instated, reset it. */ + + if (reset_caseful) + { + options &= ~PCRE2_CASELESS; + req_caseopt = 0; + reset_caseful = FALSE; + } + + break; /* End literal character handling */ } /* End of big switch */ } /* End of big loop */ @@ -7874,7 +8018,10 @@ length = 2 + 2*LINK_SIZE + skipunits; /* Remember if this is a lookbehind assertion, and if it is, save its length and skip over the pattern offset. */ -lookbehind = *code == OP_ASSERTBACK || *code == OP_ASSERTBACK_NOT; +lookbehind = *code == OP_ASSERTBACK || + *code == OP_ASSERTBACK_NOT || + *code == OP_ASSERTBACK_NA; + if (lookbehind) { lookbehindlength = META_DATA(pptr[-1]); @@ -7892,7 +8039,6 @@ if (*code == OP_CBRA) capnumber = GET2(code, 1 + LINK_SIZE); capitem.number = capnumber; capitem.next = cb->open_caps; - capitem.flag = FALSE; capitem.assert_depth = cb->assert_depth; cb->open_caps = &capitem; } @@ -7948,7 +8094,7 @@ for (;;) /* If this is not the first branch, the first char and reqcu have to match the values from all the previous branches, except that if the previous value for reqcu didn't have REQ_VARY set, it can still match, - and we set REQ_VARY for the regex. */ + and we set REQ_VARY for the group from this branch's value. */ else { @@ -7987,7 +8133,7 @@ for (;;) else { reqcu = branchreqcu; - reqcuflags |= branchreqcuflags; /* To "or" REQ_VARY */ + reqcuflags |= branchreqcuflags; /* To "or" REQ_VARY if present */ } } } @@ -8021,26 +8167,9 @@ for (;;) PUT(code, 1, (int)(code - start_bracket)); code += 1 + LINK_SIZE; - /* If it was a capturing subpattern, check to see if it contained any - recursive back references. If so, we must wrap it in atomic brackets. In - any event, remove the block from the chain. */ + /* If it was a capturing subpattern, remove the block from the chain. */ - if (capnumber > 0) - { - if (cb->open_caps->flag) - { - (void)memmove(start_bracket + 1 + LINK_SIZE, start_bracket, - CU2BYTES(code - start_bracket)); - *start_bracket = OP_ONCE; - code += 1 + LINK_SIZE; - PUT(start_bracket, 1, (int)(code - start_bracket)); - *code = OP_KET; - PUT(code, 1, (int)(code - start_bracket)); - code += 1 + LINK_SIZE; - length += 2 + 2*LINK_SIZE; - } - cb->open_caps = cb->open_caps->next; - } + if (capnumber > 0) cb->open_caps = cb->open_caps->next; /* Set values to pass back */ @@ -8167,7 +8296,7 @@ do { /* Positive forward assertion */ - else if (op == OP_ASSERT) + else if (op == OP_ASSERT || op == OP_ASSERT_NA) { if (!is_anchored(scode, bracket_map, cb, atomcount, TRUE)) return FALSE; } @@ -8305,7 +8434,7 @@ do { /* Positive forward assertions */ - else if (op == OP_ASSERT) + else if (op == OP_ASSERT || op == OP_ASSERT_NA) { if (!is_startline(scode, bracket_map, cb, atomcount, TRUE)) return FALSE; @@ -8547,9 +8676,11 @@ do { case OP_CBRAPOS: case OP_SCBRAPOS: case OP_ASSERT: + case OP_ASSERT_NA: case OP_ONCE: case OP_SCRIPT_RUN: - d = find_firstassertedcu(scode, &dflags, inassert + ((op==OP_ASSERT)?1:0)); + d = find_firstassertedcu(scode, &dflags, inassert + + ((op == OP_ASSERT || op == OP_ASSERT_NA)?1:0)); if (dflags < 0) return 0; if (cflags < 0) { c = d; cflags = dflags; } @@ -8578,6 +8709,19 @@ do { case OP_MINPLUSI: case OP_POSPLUSI: if (inassert == 0) return 0; + + /* If the character is more than one code unit long, we cannot set its + first code unit when matching caselessly. Later scanning may pick up + multiple code units. */ + +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (scode[1] >= 0x80) return 0; +#elif PCRE2_CODE_UNIT_WIDTH == 16 + if (scode[1] >= 0xd800 && scode[1] <= 0xdfff) return 0; +#endif +#endif + if (cflags < 0) { c = scode[1]; cflags = REQ_CASELESS; } else if (c != scode[1]) return 0; break; @@ -8660,9 +8804,10 @@ memset(slot + IMM2_SIZE + length, 0, /* This function is called to skip parts of the parsed pattern when finding the length of a lookbehind branch. It is called after (*ACCEPT) and (*FAIL) to find -the end of the branch, it is called to skip over an internal lookaround, and it -is also called to skip to the end of a class, during which it will never -encounter nested groups (but there's no need to have special code for that). +the end of the branch, it is called to skip over an internal lookaround or +(DEFINE) group, and it is also called to skip to the end of a class, during +which it will never encounter nested groups (but there's no need to have +special code for that). When called to find the end of a branch or group, pptr must point to the first meta code inside the branch, not the branch-starting code. In other cases it @@ -8745,8 +8890,10 @@ for (;; pptr++) case META_COND_VERSION: case META_LOOKAHEAD: case META_LOOKAHEADNOT: + case META_LOOKAHEAD_NA: case META_LOOKBEHIND: case META_LOOKBEHINDNOT: + case META_LOOKBEHIND_NA: case META_NOCAPTURE: case META_SCRIPT_RUN: nestlevel++; @@ -8798,7 +8945,7 @@ Returns: the group length or a negative number static int get_grouplength(uint32_t **pptrptr, BOOL isinline, int *errcodeptr, int *lcptr, - int group, parsed_recurse_check *recurses, compile_block *cb) + int group, parsed_recurse_check *recurses, compile_block *cb) { int branchlength; int grouplength = -1; @@ -8847,8 +8994,7 @@ return -1; *************************************************/ /* Return a fixed length for a branch in a lookbehind, giving an error if the -length is not fixed. If any lookbehinds are encountered on the way, they get -their length set. On entry, *pptrptr points to the first element inside the +length is not fixed. On entry, *pptrptr points to the first element inside the branch. On exit it is set to point to the ALT or KET. Arguments: @@ -8978,15 +9124,16 @@ for (;; pptr++) } break; - /* Lookaheads can be ignored, but we must start the skip inside the group - so that it isn't treated as a group within the branch. */ + /* Lookaheads do not contribute to the length of this branch, but they may + contain lookbehinds within them whose lengths need to be set. */ case META_LOOKAHEAD: case META_LOOKAHEADNOT: - pptr = parsed_skip(pptr + 1, PSKIP_KET); - if (pptr == NULL) goto PARSED_SKIP_FAILED; + case META_LOOKAHEAD_NA: + *errcodeptr = check_lookbehinds(pptr + 1, &pptr, recurses, cb); + if (*errcodeptr != 0) return -1; - /* Also ignore any qualifiers that follow a lookahead assertion. */ + /* Ignore any qualifiers that follow a lookahead assertion. */ switch (pptr[1]) { @@ -9013,10 +9160,12 @@ for (;; pptr++) } break; - /* Lookbehinds can be ignored, but must themselves be checked. */ + /* A nested lookbehind does not contribute any length to this lookbehind, + but must itself be checked and have its lengths set. */ case META_LOOKBEHIND: case META_LOOKBEHINDNOT: + case META_LOOKBEHIND_NA: if (!set_lookbehind_lengths(&pptr, errcodeptr, lcptr, recurses, cb)) return -1; break; @@ -9136,14 +9285,21 @@ for (;; pptr++) itemlength = grouplength; break; - /* Check nested groups - advance past the initial data for each type and - then seek a fixed length with get_grouplength(). */ + /* A (DEFINE) group is never obeyed inline and so it does not contribute to + the length of this branch. Skip from the following item to the next + unpaired ket. */ + + case META_COND_DEFINE: + pptr = parsed_skip(pptr + 1, PSKIP_KET); + break; + + /* Check other nested groups - advance past the initial data for each type + and then seek a fixed length with get_grouplength(). */ case META_COND_NAME: case META_COND_NUMBER: case META_COND_RNAME: case META_COND_RNUMBER: - case META_COND_DEFINE: pptr += 2 + SIZEOFFSET; goto CHECK_GROUP; @@ -9178,8 +9334,26 @@ for (;; pptr++) case META_MINMAX_QUERY: if (pptr[1] == pptr[2]) { - if (pptr[1] == 0) branchlength -= lastitemlength; - else itemlength = (pptr[1] - 1) * lastitemlength; + switch(pptr[1]) + { + case 0: + branchlength -= lastitemlength; + break; + + case 1: + itemlength = 0; + break; + + default: /* Check for integer overflow */ + if (lastitemlength != 0 && /* Should not occur, but just in case */ + INT_MAX/lastitemlength < pptr[1] - 1) + { + *errcodeptr = ERR87; /* Integer overflow; lookbehind too big */ + return -1; + } + itemlength = (pptr[1] - 1) * lastitemlength; + break; + } pptr += 2; break; } @@ -9193,24 +9367,23 @@ for (;; pptr++) return -1; } - /* Add the item length to the branchlength, and save it for use if the next - thing is a quantifier. */ + /* Add the item length to the branchlength, checking for integer overflow and + for the branch length exceeding the limit. */ - branchlength += itemlength; - lastitemlength = itemlength; - - /* Ensure that the length does not overflow the limit. */ - - if (branchlength > LOOKBEHIND_MAX) + if (INT_MAX - branchlength < (int)itemlength || + (branchlength += itemlength) > LOOKBEHIND_MAX) { *errcodeptr = ERR87; return -1; } + + /* Save this item length for use if the next item is a quantifier. */ + + lastitemlength = itemlength; } EXIT: *pptrptr = pptr; -if (branchlength > cb->max_lookbehind) cb->max_lookbehind = branchlength; return branchlength; PARSED_SKIP_FAILED: @@ -9229,6 +9402,11 @@ branches. An error occurs if any branch does not have a fixed length that is less than the maximum (65535). On exit, the pointer must be left on the final ket. +The function also maintains the max_lookbehind value. Any lookbehind branch +that contains a nested lookbehind may actually look further back than the +length of the branch. The additional amount is passed back from +get_branchlength() as an "extra" value. + Arguments: pptrptr pointer to pointer in the parsed pattern errcodeptr pointer to error code @@ -9262,6 +9440,7 @@ do if (cb->erroroffset == PCRE2_UNSET) cb->erroroffset = offset; return FALSE; } + if (branchlength > cb->max_lookbehind) cb->max_lookbehind = branchlength; *bptr |= branchlength; /* branchlength never more than 65535 */ bptr = *pptrptr; } @@ -9282,20 +9461,30 @@ set_lookbehind_lengths() for each one. At the start, the errorcode is zero and the error offset is marked unset. The enables the functions above not to override settings from deeper nestings. -Arguments cb points to the compile block -Returns: 0 on success, or an errorcode (cb->erroroffset will be set) +This function is called recursively from get_branchlength() for lookaheads in +order to process any lookbehinds that they may contain. It stops when it hits a +non-nested closing parenthesis in this case, returning a pointer to it. + +Arguments + pptr points to where to start (start of pattern or start of lookahead) + retptr if not NULL, return the ket pointer here + recurses chain of recurse_check to catch mutual recursion + cb points to the compile block + +Returns: 0 on success, or an errorcode (cb->erroroffset will be set) */ static int -check_lookbehinds(compile_block *cb) +check_lookbehinds(uint32_t *pptr, uint32_t **retptr, + parsed_recurse_check *recurses, compile_block *cb) { -uint32_t *pptr; int errorcode = 0; int loopcount = 0; +int nestlevel = 0; cb->erroroffset = PCRE2_UNSET; -for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) +for (; *pptr != META_END; pptr++) { if (*pptr < META_END) continue; /* Literal */ @@ -9309,14 +9498,31 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) pptr += 1; break; + case META_KET: + if (--nestlevel < 0) + { + if (retptr != NULL) *retptr = pptr; + return 0; + } + break; + + case META_ATOMIC: + case META_CAPTURE: + case META_COND_ASSERT: + case META_LOOKAHEAD: + case META_LOOKAHEADNOT: + case META_LOOKAHEAD_NA: + case META_NOCAPTURE: + case META_SCRIPT_RUN: + nestlevel++; + break; + case META_ACCEPT: case META_ALT: case META_ASTERISK: case META_ASTERISK_PLUS: case META_ASTERISK_QUERY: - case META_ATOMIC: case META_BACKREF: - case META_CAPTURE: case META_CIRCUMFLEX: case META_CLASS: case META_CLASS_EMPTY: @@ -9324,14 +9530,9 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) case META_CLASS_END: case META_CLASS_NOT: case META_COMMIT: - case META_COND_ASSERT: case META_DOLLAR: case META_DOT: case META_FAIL: - case META_KET: - case META_LOOKAHEAD: - case META_LOOKAHEADNOT: - case META_NOCAPTURE: case META_PLUS: case META_PLUS_PLUS: case META_PLUS_QUERY: @@ -9341,7 +9542,6 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) case META_QUERY_QUERY: case META_RANGE_ESCAPED: case META_RANGE_LITERAL: - case META_SCRIPT_RUN: case META_SKIP: case META_THEN: break; @@ -9351,13 +9551,26 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) break; case META_BACKREF_BYNAME: + case META_RECURSE_BYNAME: + pptr += 1 + SIZEOFFSET; + break; + case META_COND_DEFINE: + pptr += SIZEOFFSET; + nestlevel++; + break; + case META_COND_NAME: case META_COND_NUMBER: case META_COND_RNAME: case META_COND_RNUMBER: - case META_RECURSE_BYNAME: pptr += 1 + SIZEOFFSET; + nestlevel++; + break; + + case META_COND_VERSION: + pptr += 3; + nestlevel++; break; case META_CALLOUT_STRING: @@ -9378,7 +9591,6 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) break; case META_CALLOUT_NUMBER: - case META_COND_VERSION: pptr += 3; break; @@ -9392,7 +9604,8 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) case META_LOOKBEHIND: case META_LOOKBEHINDNOT: - if (!set_lookbehind_lengths(&pptr, &errorcode, &loopcount, NULL, cb)) + case META_LOOKBEHIND_NA: + if (!set_lookbehind_lengths(&pptr, &errorcode, &loopcount, recurses, cb)) return errorcode; break; } @@ -9427,6 +9640,7 @@ pcre2_compile(PCRE2_SPTR pattern, PCRE2_SIZE patlen, uint32_t options, int *errorptr, PCRE2_SIZE *erroroffset, pcre2_compile_context *ccontext) { BOOL utf; /* Set TRUE for UTF mode */ +BOOL ucp; /* Set TRUE for UCP mode */ BOOL has_lookbehind = FALSE; /* Set TRUE if a lookbehind is found */ BOOL zero_terminated; /* Set TRUE for zero-terminated pattern */ pcre2_real_code *re = NULL; /* What we will return */ @@ -9494,6 +9708,10 @@ if (pattern == NULL) if (ccontext == NULL) ccontext = (pcre2_compile_context *)(&PRIV(default_compile_context)); +/* PCRE2_MATCH_INVALID_UTF implies UTF */ + +if ((options & PCRE2_MATCH_INVALID_UTF) != 0) options |= PCRE2_UTF; + /* Check that all undefined public option bits are zero. */ if ((options & ~PUBLIC_COMPILE_OPTIONS) != 0 || @@ -9672,7 +9890,7 @@ if ((options & PCRE2_LITERAL) == 0) ptr += skipatstart; -/* Can't support UTF or UCP unless PCRE2 has been compiled with UTF support. */ +/* Can't support UTF or UCP if PCRE2 was built without Unicode support. */ #ifndef SUPPORT_UNICODE if ((cb.external_options & (PCRE2_UTF|PCRE2_UCP)) != 0) @@ -9710,8 +9928,8 @@ if (utf) /* Check UCP lockout. */ -if ((cb.external_options & (PCRE2_UCP|PCRE2_NEVER_UCP)) == - (PCRE2_UCP|PCRE2_NEVER_UCP)) +ucp = (cb.external_options & PCRE2_UCP) != 0; +if (ucp && (cb.external_options & PCRE2_NEVER_UCP) != 0) { errorcode = ERR75; goto HAD_EARLY_ERROR; @@ -9842,7 +10060,7 @@ lengths. */ if (has_lookbehind) { - errorcode = check_lookbehinds(&cb); + errorcode = check_lookbehinds(cb.parsed_pattern, NULL, NULL, &cb); if (errorcode != 0) goto HAD_CB_ERROR; } @@ -9990,8 +10208,9 @@ re->max_lookbehind = cb.max_lookbehind; if (cb.had_accept) { - reqcu = 0; /* Must disable after (*ACCEPT) */ + reqcu = 0; /* Must disable after (*ACCEPT) */ reqcuflags = REQ_NONE; + re->flags |= PCRE2_HASACCEPT; /* Disables minimum length */ } /* Fill in the final opcode and check for disastrous overflow. If no overflow, @@ -10086,7 +10305,7 @@ function call. */ if (errorcode == 0 && (re->overall_options & PCRE2_NO_AUTO_POSSESS) == 0) { PCRE2_UCHAR *temp = (PCRE2_UCHAR *)codestart; - if (PRIV(auto_possessify)(temp, utf, &cb) != 0) errorcode = ERR80; + if (PRIV(auto_possessify)(temp, &cb) != 0) errorcode = ERR80; } /* Failed to compile, or error while post-processing. */ @@ -10112,6 +10331,8 @@ unit. */ if ((re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0) { + int minminlength = 0; /* For minimal minlength from first/required CU */ + /* If we do not have a first code unit, see if there is one that is asserted (these are not saved during the compile because they can cause conflicts with actual literals that follow). */ @@ -10119,32 +10340,38 @@ if ((re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0) if (firstcuflags < 0) firstcu = find_firstassertedcu(codestart, &firstcuflags, 0); - /* Save the data for a first code unit. */ + /* Save the data for a first code unit. The existence of one means the + minimum length must be at least 1. */ if (firstcuflags >= 0) { re->first_codeunit = firstcu; re->flags |= PCRE2_FIRSTSET; + minminlength++; /* Handle caseless first code units. */ if ((firstcuflags & REQ_CASELESS) != 0) { - if (firstcu < 128 || (!utf && firstcu < 255)) + if (firstcu < 128 || (!utf && !ucp && firstcu < 255)) { if (cb.fcc[firstcu] != firstcu) re->flags |= PCRE2_FIRSTCASELESS; } - /* The first code unit is > 128 in UTF mode, or > 255 otherwise. In - 8-bit UTF mode, codepoints in the range 128-255 are introductory code - points and cannot have another case. In 16-bit and 32-bit modes, we can - check wide characters when UTF (and therefore UCP) is supported. */ + /* The first code unit is > 128 in UTF or UCP mode, or > 255 otherwise. + In 8-bit UTF mode, codepoints in the range 128-255 are introductory code + points and cannot have another case, but if UCP is set they may do. */ -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - else if (firstcu <= MAX_UTF_CODE_POINT && +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + else if (ucp && !utf && UCD_OTHERCASE(firstcu) != firstcu) + re->flags |= PCRE2_FIRSTCASELESS; +#else + else if ((utf || ucp) && firstcu <= MAX_UTF_CODE_POINT && UCD_OTHERCASE(firstcu) != firstcu) re->flags |= PCRE2_FIRSTCASELESS; #endif +#endif /* SUPPORT_UNICODE */ } } @@ -10158,39 +10385,78 @@ if ((re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0) is_startline(codestart, 0, &cb, 0, FALSE)) re->flags |= PCRE2_STARTLINE; - /* Handle the "required code unit", if one is set. In the case of an anchored - pattern, do this only if it follows a variable length item in the pattern. */ + /* Handle the "required code unit", if one is set. In the UTF case we can + increment the minimum minimum length only if we are sure this really is a + different character and not a non-starting code unit of the first character, + because the minimum length count is in characters, not code units. */ - if (reqcuflags >= 0 && - ((re->overall_options & PCRE2_ANCHORED) == 0 || - (reqcuflags & REQ_VARY) != 0)) + if (reqcuflags >= 0) { - re->last_codeunit = reqcu; - re->flags |= PCRE2_LASTSET; +#if PCRE2_CODE_UNIT_WIDTH == 16 + if ((re->overall_options & PCRE2_UTF) == 0 || /* Not UTF */ + firstcuflags < 0 || /* First not set */ + (firstcu & 0xf800) != 0xd800 || /* First not surrogate */ + (reqcu & 0xfc00) != 0xdc00) /* Req not low surrogate */ +#elif PCRE2_CODE_UNIT_WIDTH == 8 + if ((re->overall_options & PCRE2_UTF) == 0 || /* Not UTF */ + firstcuflags < 0 || /* First not set */ + (firstcu & 0x80) == 0 || /* First is ASCII */ + (reqcu & 0x80) == 0) /* Req is ASCII */ +#endif + { + minminlength++; + } - /* Handle caseless required code units as for first code units (above). */ + /* In the case of an anchored pattern, set up the value only if it follows + a variable length item in the pattern. */ - if ((reqcuflags & REQ_CASELESS) != 0) + if ((re->overall_options & PCRE2_ANCHORED) == 0 || + (reqcuflags & REQ_VARY) != 0) { - if (reqcu < 128 || (!utf && reqcu < 255)) + re->last_codeunit = reqcu; + re->flags |= PCRE2_LASTSET; + + /* Handle caseless required code units as for first code units (above). */ + + if ((reqcuflags & REQ_CASELESS) != 0) { - if (cb.fcc[reqcu] != reqcu) re->flags |= PCRE2_LASTCASELESS; - } -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - else if (reqcu <= MAX_UTF_CODE_POINT && UCD_OTHERCASE(reqcu) != reqcu) + if (reqcu < 128 || (!utf && !ucp && reqcu < 255)) + { + if (cb.fcc[reqcu] != reqcu) re->flags |= PCRE2_LASTCASELESS; + } +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + else if (ucp && !utf && UCD_OTHERCASE(reqcu) != reqcu) + re->flags |= PCRE2_LASTCASELESS; +#else + else if ((utf || ucp) && reqcu <= MAX_UTF_CODE_POINT && + UCD_OTHERCASE(reqcu) != reqcu) re->flags |= PCRE2_LASTCASELESS; #endif +#endif /* SUPPORT_UNICODE */ + } } } - /* Finally, study the compiled pattern to set up information such as a bitmap - of starting code units and a minimum matching length. */ + /* Study the compiled pattern to set up information such as a bitmap of + starting code units and a minimum matching length. */ if (PRIV(study)(re) != 0) { errorcode = ERR31; goto HAD_CB_ERROR; } + + /* If study() set a bitmap of starting code units, it implies a minimum + length of at least one. */ + + if ((re->flags & PCRE2_FIRSTMAPSET) != 0 && minminlength == 0) + minminlength = 1; + + /* If the minimum length set (or not set) by study() is less than the minimum + implied by required code units, override it. */ + + if (re->minlength < minminlength) re->minlength = minminlength; } /* End of start-of-match optimizations. */ /* Control ends up here in all cases. When running under valgrind, make a diff --git a/dist2/src/pcre2_config.c b/dist2/src/pcre2_config.c index e487b102..5ef103ca 100644 --- a/dist2/src/pcre2_config.c +++ b/dist2/src/pcre2_config.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2017 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -43,7 +43,8 @@ POSSIBILITY OF SUCH DAMAGE. #endif /* Save the configured link size, which is in bytes. In 16-bit and 32-bit modes -its value gets changed by pcre2_internal.h to be in code units. */ +its value gets changed by pcre2_intmodedep.h (included by pcre2_internal.h) to +be in code units. */ static int configured_link_size = LINK_SIZE; @@ -94,6 +95,7 @@ if (where == NULL) /* Requests a length */ case PCRE2_CONFIG_NEWLINE: case PCRE2_CONFIG_PARENSLIMIT: case PCRE2_CONFIG_STACKRECURSE: /* Obsolete */ + case PCRE2_CONFIG_TABLES_LENGTH: case PCRE2_CONFIG_UNICODE: return sizeof(uint32_t); @@ -191,6 +193,10 @@ switch (what) *((uint32_t *)where) = 0; break; + case PCRE2_CONFIG_TABLES_LENGTH: + *((uint32_t *)where) = TABLES_LENGTH; + break; + case PCRE2_CONFIG_UNICODE_VERSION: { #if defined SUPPORT_UNICODE diff --git a/dist2/src/pcre2_context.c b/dist2/src/pcre2_context.c index 9c2886a6..f904a494 100644 --- a/dist2/src/pcre2_context.c +++ b/dist2/src/pcre2_context.c @@ -323,7 +323,7 @@ data. */ PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION pcre2_set_character_tables(pcre2_compile_context *ccontext, - const unsigned char *tables) + const uint8_t *tables) { ccontext->tables = tables; return 0; diff --git a/dist2/src/pcre2_dfa_match.c b/dist2/src/pcre2_dfa_match.c index bbf3e210..625695b7 100644 --- a/dist2/src/pcre2_dfa_match.c +++ b/dist2/src/pcre2_dfa_match.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -173,6 +173,8 @@ static const uint8_t coptable[] = { 0, /* Assert not */ 0, /* Assert behind */ 0, /* Assert behind not */ + 0, /* NA assert */ + 0, /* NA assert behind */ 0, /* ONCE */ 0, /* SCRIPT_RUN */ 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ @@ -248,6 +250,8 @@ static const uint8_t poptable[] = { 0, /* Assert not */ 0, /* Assert behind */ 0, /* Assert behind not */ + 0, /* NA assert */ + 0, /* NA assert behind */ 0, /* ONCE */ 0, /* SCRIPT_RUN */ 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ @@ -544,6 +548,7 @@ PCRE2_SPTR start_code = mb->start_code; #ifdef SUPPORT_UNICODE BOOL utf = (mb->poptions & PCRE2_UTF) != 0; +BOOL utf_or_ucp = utf || (mb->poptions & PCRE2_UCP) != 0; #else BOOL utf = FALSE; #endif @@ -962,7 +967,7 @@ for (;;) if (ptr >= end_subject) { if ((mb->moptions & PCRE2_PARTIAL_HARD) != 0) - could_continue = TRUE; + return PCRE2_ERROR_PARTIAL; else { ADD_ACTIVE(state_offset + 1, 0); } } break; @@ -1011,10 +1016,12 @@ for (;;) /*-----------------------------------------------------------------*/ case OP_EODN: - if (clen == 0 && (mb->moptions & PCRE2_PARTIAL_HARD) != 0) - could_continue = TRUE; - else if (clen == 0 || (IS_NEWLINE(ptr) && ptr == end_subject - mb->nllen)) - { ADD_ACTIVE(state_offset + 1, 0); } + if (clen == 0 || (IS_NEWLINE(ptr) && ptr == end_subject - mb->nllen)) + { + if ((mb->moptions & PCRE2_PARTIAL_HARD) != 0) + return PCRE2_ERROR_PARTIAL; + ADD_ACTIVE(state_offset + 1, 0); + } break; /*-----------------------------------------------------------------*/ @@ -2184,7 +2191,7 @@ for (;;) if (clen == 0) break; #ifdef SUPPORT_UNICODE - if (utf) + if (utf_or_ucp) { if (c == d) { ADD_NEW(state_offset + dlen + 1, 0); } else { @@ -2198,7 +2205,7 @@ for (;;) } else #endif /* SUPPORT_UNICODE */ - /* Not UTF mode */ + /* Not UTF or UCP mode */ { if (TABLE_GET(c, lcc, c) == TABLE_GET(d, lcc, d)) { ADD_NEW(state_offset + 2, 0); } @@ -2333,7 +2340,7 @@ for (;;) { uint32_t otherd; #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -2368,7 +2375,7 @@ for (;;) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -2411,7 +2418,7 @@ for (;;) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -2452,7 +2459,7 @@ for (;;) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -2485,7 +2492,7 @@ for (;;) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -2525,7 +2532,7 @@ for (;;) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf && d >= 128) + if (utf_or_ucp && d >= 128) otherd = UCD_OTHERCASE(d); else #endif /* SUPPORT_UNICODE */ @@ -3152,8 +3159,8 @@ for (;;) /* We have finished the processing at the current subject character. If no new states have been set for the next character, we have found all the - matches that we are going to find. If we are at the top level and partial - matching has been requested, check for appropriate conditions. + matches that we are going to find. If partial matching has been requested, + check for appropriate conditions. The "forced_ fail" variable counts the number of (*F) encountered for the character. If it is equal to the original active_count (saved in @@ -3165,22 +3172,24 @@ for (;;) if (new_count <= 0) { - if (rlevel == 1 && /* Top level, and */ - could_continue && /* Some could go on, and */ + if (could_continue && /* Some could go on, and */ forced_fail != workspace[1] && /* Not all forced fail & */ ( /* either... */ (mb->moptions & PCRE2_PARTIAL_HARD) != 0 /* Hard partial */ || /* or... */ ((mb->moptions & PCRE2_PARTIAL_SOFT) != 0 && /* Soft partial and */ - match_count < 0) /* no matches */ + match_count < 0) /* no matches */ ) && /* And... */ ( - partial_newline || /* Either partial NL */ - ( /* or ... */ - ptr >= end_subject && /* End of subject and */ - ptr > mb->start_used_ptr) /* Inspected non-empty string */ + partial_newline || /* Either partial NL */ + ( /* or ... */ + ptr >= end_subject && /* End of subject and */ + ( /* either */ + ptr > mb->start_used_ptr || /* Inspected non-empty string */ + mb->allowemptypartial /* or pattern has lookbehind */ + ) /* or could match empty */ ) - ) + )) match_count = PCRE2_ERROR_PARTIAL; break; /* Exit from loop along the subject string */ } @@ -3246,6 +3255,11 @@ BOOL utf, anchored, startline, firstline; BOOL has_first_cu = FALSE; BOOL has_req_cu = FALSE; +#if PCRE2_CODE_UNIT_WIDTH == 8 +BOOL memchr_not_found_first_cu = FALSE; +BOOL memchr_not_found_first_cu2 = FALSE; +#endif + PCRE2_UCHAR first_cu = 0; PCRE2_UCHAR first_cu2 = 0; PCRE2_UCHAR req_cu = 0; @@ -3295,6 +3309,11 @@ if ((options & (PCRE2_PARTIAL_HARD|PCRE2_PARTIAL_SOFT)) != 0 && ((re->overall_options | options) & PCRE2_ENDANCHORED) != 0) return PCRE2_ERROR_BADOPTION; +/* Invalid UTF support is not available for DFA matching. */ + +if ((re->overall_options & PCRE2_MATCH_INVALID_UTF) != 0) + return PCRE2_ERROR_DFA_UINVALID_UTF; + /* Check that the first field in the block is the magic number. If it is not, return with PCRE2_ERROR_BADMAGIC. */ @@ -3404,6 +3423,8 @@ mb->tables = re->tables; mb->start_subject = subject; mb->end_subject = end_subject; mb->start_offset = start_offset; +mb->allowemptypartial = (re->max_lookbehind > 0) || + (re->flags & PCRE2_MATCH_EMPTY) != 0; mb->moptions = options; mb->poptions = re->overall_options; mb->match_call_count = 0; @@ -3506,10 +3527,15 @@ if ((re->flags & PCRE2_FIRSTSET) != 0) if ((re->flags & PCRE2_FIRSTCASELESS) != 0) { first_cu2 = TABLE_GET(first_cu, mb->tables + fcc_offset, first_cu); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (utf && first_cu > 127) +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (first_cu > 127 && !utf && (re->overall_options & PCRE2_UCP) != 0) + first_cu2 = (PCRE2_UCHAR)UCD_OTHERCASE(first_cu); +#else + if (first_cu > 127 && (utf || (re->overall_options & PCRE2_UCP) != 0)) first_cu2 = (PCRE2_UCHAR)UCD_OTHERCASE(first_cu); #endif +#endif /* SUPPORT_UNICODE */ } } else @@ -3525,9 +3551,15 @@ if ((re->flags & PCRE2_LASTSET) != 0) if ((re->flags & PCRE2_LASTCASELESS) != 0) { req_cu2 = TABLE_GET(req_cu, mb->tables + fcc_offset, req_cu); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (utf && req_cu > 127) req_cu2 = (PCRE2_UCHAR)UCD_OTHERCASE(req_cu); +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (req_cu > 127 && !utf && (re->overall_options & PCRE2_UCP) != 0) + req_cu2 = (PCRE2_UCHAR)UCD_OTHERCASE(req_cu); +#else + if (req_cu > 127 && (utf || (re->overall_options & PCRE2_UCP) != 0)) + req_cu2 = (PCRE2_UCHAR)UCD_OTHERCASE(req_cu); #endif +#endif /* SUPPORT_UNICODE */ } } @@ -3619,7 +3651,10 @@ for (;;) /* Not anchored. Advance to a unique first code unit if there is one. In 8-bit mode, the use of memchr() gives a big speed up, even though we have to call it twice in caseless mode, in order to find the earliest occurrence - of the character in either of its cases. */ + of the character in either of its cases. If a call to memchr() that + searches the rest of the subject fails to find one case, remember that in + order not to keep on repeating the search. This can make a huge difference + when the strings are very long and only one case is present. */ else { @@ -3633,11 +3668,29 @@ for (;;) (smc = UCHAR21TEST(start_match)) != first_cu && smc != first_cu2) start_match++; + #else /* 8-bit code units */ - PCRE2_SPTR pp1 = - memchr(start_match, first_cu, end_subject-start_match); - PCRE2_SPTR pp2 = - memchr(start_match, first_cu2, end_subject-start_match); + PCRE2_SPTR pp1 = NULL; + PCRE2_SPTR pp2 = NULL; + PCRE2_SIZE cu2size = end_subject - start_match; + + if (!memchr_not_found_first_cu) + { + pp1 = memchr(start_match, first_cu, end_subject - start_match); + if (pp1 == NULL) memchr_not_found_first_cu = TRUE; + else cu2size = pp1 - start_match; + } + + /* If pp1 is not NULL, we have arranged to search only as far as pp1, + to see if the other case is earlier, so we can set "not found" only + when both searches have returned NULL. */ + + if (!memchr_not_found_first_cu2) + { + pp2 = memchr(start_match, first_cu2, cu2size); + memchr_not_found_first_cu2 = (pp2 == NULL && pp1 == NULL); + } + if (pp1 == NULL) start_match = (pp2 == NULL)? end_subject : pp2; else @@ -3653,7 +3706,7 @@ for (;;) while (start_match < end_subject && UCHAR21TEST(start_match) != first_cu) start_match++; -#else +#else /* 8-bit code units */ start_match = memchr(start_match, first_cu, end_subject - start_match); if (start_match == NULL) start_match = end_subject; #endif @@ -3740,6 +3793,8 @@ for (;;) if ((mb->moptions & (PCRE2_PARTIAL_HARD|PCRE2_PARTIAL_SOFT)) == 0) { + PCRE2_SPTR p; + /* The minimum matching length is a lower bound; no actual string of that length may actually match the pattern. Although the value is, strictly, in characters, we treat it as code units to avoid spending too much time @@ -3753,37 +3808,63 @@ for (;;) point. This optimization can save a huge amount of backtracking in patterns with nested unlimited repeats that aren't going to match. Writing separate code for cased/caseless versions makes it go faster, as - does using an autoincrement and backing off on a match. + does using an autoincrement and backing off on a match. As in the case of + the first code unit, using memchr() in the 8-bit library gives a big + speed up. Unlike the first_cu check above, we do not need to call + memchr() twice in the caseless case because we only need to check for the + presence of the character in either case, not find the first occurrence. + + The search can be skipped if the code unit was found later than the + current starting point in a previous iteration of the bumpalong loop. HOWEVER: when the subject string is very, very long, searching to its end can take a long time, and give bad performance on quite ordinary patterns. This showed up when somebody was matching something like /^\d+C/ on a 32-megabyte string... so we don't do this when the string is - sufficiently long. */ + sufficiently long, but it's worth searching a lot more for unanchored + patterns. */ - if (has_req_cu && end_subject - start_match < REQ_CU_MAX) + p = start_match + (has_first_cu? 1:0); + if (has_req_cu && p > req_cu_ptr) { - PCRE2_SPTR p = start_match + (has_first_cu? 1:0); + PCRE2_SIZE check_length = end_subject - start_match; - /* We don't need to repeat the search if we haven't yet reached the - place we found it at last time. */ - - if (p > req_cu_ptr) + if (check_length < REQ_CU_MAX || + (!anchored && check_length < REQ_CU_MAX * 1000)) { - if (req_cu != req_cu2) + if (req_cu != req_cu2) /* Caseless */ { +#if PCRE2_CODE_UNIT_WIDTH != 8 while (p < end_subject) { uint32_t pp = UCHAR21INCTEST(p); if (pp == req_cu || pp == req_cu2) { p--; break; } } +#else /* 8-bit code units */ + PCRE2_SPTR pp = p; + p = memchr(pp, req_cu, end_subject - pp); + if (p == NULL) + { + p = memchr(pp, req_cu2, end_subject - pp); + if (p == NULL) p = end_subject; + } +#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */ } + + /* The caseful case */ + else { +#if PCRE2_CODE_UNIT_WIDTH != 8 while (p < end_subject) { if (UCHAR21INCTEST(p) == req_cu) { p--; break; } } + +#else /* 8-bit code units */ + p = memchr(p, req_cu, end_subject - p); + if (p == NULL) p = end_subject; +#endif } /* If we can't find the required code unit, break the matching loop, diff --git a/dist2/src/dftables.c b/dist2/src/pcre2_dftables.c index 02796cc6..71b90ce8 100644 --- a/dist2/src/dftables.c +++ b/dist2/src/pcre2_dftables.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -40,9 +40,12 @@ POSSIBILITY OF SUCH DAMAGE. /* This is a freestanding support program to generate a file containing -character tables for PCRE2. The tables are built according to the current -locale using the pcre2_maketables() function, which is part of the PCRE2 API. -*/ +character tables for PCRE2. The tables are built using the pcre2_maketables() +function, which is part of the PCRE2 API. By default, the system's "C" locale +is used rather than what the building user happens to have set, but the -L +option can be used to select the current locale from the LC_ALL environment +variable. By default, the tables are written in source form, but if -b is +given, they are written in binary. */ #ifdef HAVE_CONFIG_H #include "config.h" @@ -56,73 +59,153 @@ locale using the pcre2_maketables() function, which is part of the PCRE2 API. #define PCRE2_CODE_UNIT_WIDTH 0 /* Must be set, but not relevant here */ #include "pcre2_internal.h" -#define DFTABLES /* pcre2_maketables.c notices this */ +#define PCRE2_DFTABLES /* pcre2_maketables.c notices this */ #include "pcre2_maketables.c" + +static const char *classlist[] = + { + "space", "xdigit", "digit", "upper", "lower", + "word", "graph", "print", "punct", "cntrl" + }; + + + +/************************************************* +* Usage * +*************************************************/ + +static void +usage(void) +{ +(void)fprintf(stderr, + "Usage: pcre2_dftables [options] <output file>\n" + " -b Write output in binary (default is source code)\n" + " -L Use locale from LC_ALL (default is \"C\" locale)\n" + ); +} + + + +/************************************************* +* Entry point * +*************************************************/ + int main(int argc, char **argv) { FILE *f; -int i = 1; +int i; +int nclass = 0; +BOOL binary = FALSE; +char *env = (char *)"C"; const unsigned char *tables; const unsigned char *base_of_tables; -/* By default, the default C locale is used rather than what the building user -happens to have set. However, if the -L option is given, set the locale from -the LC_xxx environment variables. */ +/* Process options */ -if (argc > 1 && strcmp(argv[1], "-L") == 0) +for (i = 1; i < argc; i++) { - setlocale(LC_ALL, ""); /* Set from environment variables */ - i++; + char *arg = argv[i]; + if (*arg != '-') break; + + if (strcmp(arg, "-help") == 0 || strcmp(arg, "--help") == 0) + { + usage(); + return 0; + } + + else if (strcmp(arg, "-L") == 0) + { + if (setlocale(LC_ALL, "") == NULL) + { + (void)fprintf(stderr, "pcre2_dftables: setlocale() failed\n"); + return 1; + } + env = getenv("LC_ALL"); + } + + else if (strcmp(arg, "-b") == 0) + binary = TRUE; + + else + { + (void)fprintf(stderr, "pcre2_dftables: unrecognized option %s\n", arg); + return 1; + } } -if (argc < i + 1) +if (i != argc - 1) { - fprintf(stderr, "dftables: one filename argument is required\n"); + (void)fprintf(stderr, "pcre2_dftables: one filename argument is required\n"); return 1; } +/* Make the tables */ + tables = maketables(); base_of_tables = tables; f = fopen(argv[i], "wb"); if (f == NULL) { - fprintf(stderr, "dftables: failed to open %s for writing\n", argv[1]); + fprintf(stderr, "pcre2_dftables: failed to open %s for writing\n", argv[1]); return 1; } -/* There are several fprintf() calls here, because gcc in pedantic mode -complains about the very long string otherwise. */ +/* If -b was specified, we write the tables in binary. */ -fprintf(f, +if (binary) + { + int yield = 0; + size_t len = fwrite(tables, 1, TABLES_LENGTH, f); + if (len != TABLES_LENGTH) + { + (void)fprintf(stderr, "pcre2_dftables: fwrite() returned wrong length %d " + "instead of %d\n", (int)len, TABLES_LENGTH); + yield = 1; + } + fclose(f); + free((void *)base_of_tables); + return yield; + } + +/* Write the tables as source code for inclusion in the PCRE2 library. There +are several fprintf() calls here, because gcc in pedantic mode complains about +the very long string otherwise. */ + +(void)fprintf(f, "/*************************************************\n" "* Perl-Compatible Regular Expressions *\n" "*************************************************/\n\n" - "/* This file was automatically written by the dftables auxiliary\n" + "/* This file was automatically written by the pcre2_dftables auxiliary\n" "program. It contains character tables that are used when no external\n" "tables are passed to PCRE2 by the application that calls it. The tables\n" "are used only for characters whose code values are less than 256. */\n\n"); -fprintf(f, - "/*The dftables program (which is distributed with PCRE2) can be used to\n" - "build alternative versions of this file. This is necessary if you are\n" +(void)fprintf(f, + "/* This set of tables was written in the %s locale. */\n\n", env); + +(void)fprintf(f, + "/* The pcre2_ftables program (which is distributed with PCRE2) can be used\n" + "to build alternative versions of this file. This is necessary if you are\n" "running in an EBCDIC environment, or if you want to default to a different\n" - "encoding, for example ISO-8859-1. When dftables is run, it creates these\n" - "tables in the current locale. This happens automatically if PCRE2 is\n" - "configured with --enable-rebuild-chartables. */\n\n"); + "encoding, for example ISO-8859-1. When pcre2_dftables is run, it creates\n" + "these tables in the \"C\" locale by default. This happens automatically if\n" + "PCRE2 is configured with --enable-rebuild-chartables. However, you can run\n" + "pcre2_dftables manually with the -L option to build tables using the LC_ALL\n" + "locale. */\n\n"); /* Force config.h in z/OS */ #if defined NATIVE_ZOS -fprintf(f, +(void)fprintf(f, "/* For z/OS, config.h is forced */\n" "#ifndef HAVE_CONFIG_H\n" "#define HAVE_CONFIG_H 1\n" "#endif\n\n"); #endif -fprintf(f, +(void)fprintf(f, "/* The following #include is present because without it gcc 4.x may remove\n" "the array definition from the final binary if PCRE2 is built into a static\n" "library and dead code stripping is activated. This leads to link errors.\n" @@ -130,56 +213,57 @@ fprintf(f, "outside this compilation unit might reference this\" and so it will always\n" "be supplied to the linker. */\n\n"); -fprintf(f, +(void)fprintf(f, "#ifdef HAVE_CONFIG_H\n" "#include \"config.h\"\n" "#endif\n\n" "#include \"pcre2_internal.h\"\n\n"); -fprintf(f, +(void)fprintf(f, "const uint8_t PRIV(default_tables)[] = {\n\n" "/* This table is a lower casing table. */\n\n"); -fprintf(f, " "); +(void)fprintf(f, " "); for (i = 0; i < 256; i++) { if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); fprintf(f, "%3d", *tables++); if (i != 255) fprintf(f, ","); } -fprintf(f, ",\n\n"); +(void)fprintf(f, ",\n\n"); -fprintf(f, "/* This table is a case flipping table. */\n\n"); +(void)fprintf(f, "/* This table is a case flipping table. */\n\n"); -fprintf(f, " "); +(void)fprintf(f, " "); for (i = 0; i < 256; i++) { if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); fprintf(f, "%3d", *tables++); if (i != 255) fprintf(f, ","); } -fprintf(f, ",\n\n"); +(void)fprintf(f, ",\n\n"); -fprintf(f, +(void)fprintf(f, "/* This table contains bit maps for various character classes. Each map is 32\n" "bytes long and the bits run from the least significant end of each byte. The\n" "classes that have their own maps are: space, xdigit, digit, upper, lower, word,\n" - "graph print, punct, and cntrl. Other classes are built from combinations. */\n\n"); + "graph, print, punct, and cntrl. Other classes are built from combinations. */\n\n"); -fprintf(f, " "); +(void)fprintf(f, " "); for (i = 0; i < cbit_length; i++) { if ((i & 7) == 0 && i != 0) { - if ((i & 31) == 0) fprintf(f, "\n"); - fprintf(f, "\n "); + if ((i & 31) == 0) (void)fprintf(f, "\n"); + if ((i & 24) == 8) (void)fprintf(f, " /* %s */", classlist[nclass++]); + (void)fprintf(f, "\n "); } - fprintf(f, "0x%02x", *tables++); - if (i != cbit_length - 1) fprintf(f, ","); + (void)fprintf(f, "0x%02x", *tables++); + if (i != cbit_length - 1) (void)fprintf(f, ","); } -fprintf(f, ",\n\n"); +(void)fprintf(f, ",\n\n"); -fprintf(f, +(void)fprintf(f, "/* This table identifies various classes of character by individual bits:\n" " 0x%02x white space character\n" " 0x%02x letter\n" @@ -188,32 +272,32 @@ fprintf(f, " 0x%02x alphanumeric or '_'\n*/\n\n", ctype_space, ctype_letter, ctype_lcletter, ctype_digit, ctype_word); -fprintf(f, " "); +(void)fprintf(f, " "); for (i = 0; i < 256; i++) { if ((i & 7) == 0 && i != 0) { - fprintf(f, " /* "); - if (isprint(i-8)) fprintf(f, " %c -", i-8); - else fprintf(f, "%3d-", i-8); - if (isprint(i-1)) fprintf(f, " %c ", i-1); - else fprintf(f, "%3d", i-1); - fprintf(f, " */\n "); + (void)fprintf(f, " /* "); + if (isprint(i-8)) (void)fprintf(f, " %c -", i-8); + else (void)fprintf(f, "%3d-", i-8); + if (isprint(i-1)) (void)fprintf(f, " %c ", i-1); + else (void)fprintf(f, "%3d", i-1); + (void)fprintf(f, " */\n "); } - fprintf(f, "0x%02x", *tables++); - if (i != 255) fprintf(f, ","); + (void)fprintf(f, "0x%02x", *tables++); + if (i != 255) (void)fprintf(f, ","); } -fprintf(f, "};/* "); -if (isprint(i-8)) fprintf(f, " %c -", i-8); - else fprintf(f, "%3d-", i-8); -if (isprint(i-1)) fprintf(f, " %c ", i-1); - else fprintf(f, "%3d", i-1); -fprintf(f, " */\n\n/* End of pcre2_chartables.c */\n"); +(void)fprintf(f, "};/* "); +if (isprint(i-8)) (void)fprintf(f, " %c -", i-8); + else (void)fprintf(f, "%3d-", i-8); +if (isprint(i-1)) (void)fprintf(f, " %c ", i-1); + else (void)fprintf(f, "%3d", i-1); +(void)fprintf(f, " */\n\n/* End of pcre2_chartables.c */\n"); fclose(f); free((void *)base_of_tables); return 0; } -/* End of dftables.c */ +/* End of pcre2_dftables.c */ diff --git a/dist2/src/pcre2_error.c b/dist2/src/pcre2_error.c index 1d02cf14..c61648cb 100644 --- a/dist2/src/pcre2_error.c +++ b/dist2/src/pcre2_error.c @@ -184,6 +184,8 @@ static const unsigned char compile_error_texts[] = /* 95 */ "(*alpha_assertion) not recognized\0" "script runs require Unicode support, which this version of PCRE2 does not have\0" + "too many capturing groups (maximum 65535)\0" + "atomic assertion expected after (?( or (?(?C)\0" ; /* Match-time and UTF error texts are in the same format. */ @@ -268,6 +270,7 @@ static const unsigned char match_error_texts[] = "invalid syntax\0" /* 65 */ "internal error - duplicate substitution match\0" + "PCRE2_MATCH_INVALID_UTF is not supported for DFA matching\0" ; diff --git a/dist2/src/pcre2_internal.h b/dist2/src/pcre2_internal.h index aa7ce887..d8fad1e9 100644 --- a/dist2/src/pcre2_internal.h +++ b/dist2/src/pcre2_internal.h @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -528,6 +528,7 @@ bytes in a code unit in that mode. */ #define PCRE2_HASBKPORX 0x00100000 /* contains \P, \p, or \X */ #define PCRE2_DUPCAPUSED 0x00200000 /* contains (?| */ #define PCRE2_HASBKC 0x00400000 /* contains \C */ +#define PCRE2_HASACCEPT 0x00800000 /* contains (*ACCEPT) */ #define PCRE2_MODE_MASK (PCRE2_MODE8 | PCRE2_MODE16 | PCRE2_MODE32) @@ -546,13 +547,14 @@ enum { PCRE2_MATCHEDBY_INTERPRETER, /* pcre2_match() */ #define MAGIC_NUMBER 0x50435245UL /* 'PCRE' */ /* The maximum remaining length of subject we are prepared to search for a -req_unit match. In 8-bit mode, memchr() is used and is much faster than the -search loop that has to be used in 16-bit and 32-bit modes. */ +req_unit match from an anchored pattern. In 8-bit mode, memchr() is used and is +much faster than the search loop that has to be used in 16-bit and 32-bit +modes. */ #if PCRE2_CODE_UNIT_WIDTH == 8 -#define REQ_CU_MAX 2000 +#define REQ_CU_MAX 5000 #else -#define REQ_CU_MAX 1000 +#define REQ_CU_MAX 2000 #endif /* Offsets for the bitmap tables in the cbits set of tables. Each table @@ -588,7 +590,7 @@ total length of the tables. */ #define fcc_offset 256 /* Flip case */ #define cbits_offset 512 /* Character classes */ #define ctypes_offset (cbits_offset + cbit_length) /* Character types */ -#define tables_length (ctypes_offset + 256) +#define TABLES_LENGTH (ctypes_offset + 256) /* -------------------- Character and string names ------------------------ */ @@ -892,12 +894,16 @@ a positive value. */ #define STRING_atomic0 "atomic\0" #define STRING_pla0 "pla\0" #define STRING_plb0 "plb\0" +#define STRING_napla0 "napla\0" +#define STRING_naplb0 "naplb\0" #define STRING_nla0 "nla\0" #define STRING_nlb0 "nlb\0" #define STRING_sr0 "sr\0" #define STRING_asr0 "asr\0" #define STRING_positive_lookahead0 "positive_lookahead\0" #define STRING_positive_lookbehind0 "positive_lookbehind\0" +#define STRING_non_atomic_positive_lookahead0 "non_atomic_positive_lookahead\0" +#define STRING_non_atomic_positive_lookbehind0 "non_atomic_positive_lookbehind\0" #define STRING_negative_lookahead0 "negative_lookahead\0" #define STRING_negative_lookbehind0 "negative_lookbehind\0" #define STRING_script_run0 "script_run\0" @@ -1182,12 +1188,16 @@ only. */ #define STRING_atomic0 STR_a STR_t STR_o STR_m STR_i STR_c "\0" #define STRING_pla0 STR_p STR_l STR_a "\0" #define STRING_plb0 STR_p STR_l STR_b "\0" +#define STRING_napla0 STR_n STR_a STR_p STR_l STR_a "\0" +#define STRING_naplb0 STR_n STR_a STR_p STR_l STR_b "\0" #define STRING_nla0 STR_n STR_l STR_a "\0" #define STRING_nlb0 STR_n STR_l STR_b "\0" #define STRING_sr0 STR_s STR_r "\0" #define STRING_asr0 STR_a STR_s STR_r "\0" #define STRING_positive_lookahead0 STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_a STR_h STR_e STR_a STR_d "\0" #define STRING_positive_lookbehind0 STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_b STR_e STR_h STR_i STR_n STR_d "\0" +#define STRING_non_atomic_positive_lookahead0 STR_n STR_o STR_n STR_UNDERSCORE STR_a STR_t STR_o STR_m STR_i STR_c STR_UNDERSCORE STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_a STR_h STR_e STR_a STR_d "\0" +#define STRING_non_atomic_positive_lookbehind0 STR_n STR_o STR_n STR_UNDERSCORE STR_a STR_t STR_o STR_m STR_i STR_c STR_UNDERSCORE STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_b STR_e STR_h STR_i STR_n STR_d "\0" #define STRING_negative_lookahead0 STR_n STR_e STR_g STR_a STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_a STR_h STR_e STR_a STR_d "\0" #define STRING_negative_lookbehind0 STR_n STR_e STR_g STR_a STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_b STR_e STR_h STR_i STR_n STR_d "\0" #define STRING_script_run0 STR_s STR_c STR_r STR_i STR_p STR_t STR_UNDERSCORE STR_r STR_u STR_n "\0" @@ -1312,7 +1322,7 @@ enum { ESC_A = 1, ESC_G, ESC_K, ESC_B, ESC_b, ESC_D, ESC_d, ESC_S, ESC_s, Starting from 1 (i.e. after OP_END), the values up to OP_EOD must correspond in order to the list of escapes immediately above. Furthermore, values up to OP_DOLLM must not be changed without adjusting the table called autoposstab in -pcre2_auto_possess.c +pcre2_auto_possess.c. Whenever this list is updated, the two macro definitions that follow must be updated to match. The possessification table called "opcode_possessify" in @@ -1510,80 +1520,81 @@ enum { OP_KETRMIN, /* 123 order. They are for groups the repeat for ever. */ OP_KETRPOS, /* 124 Possessive unlimited repeat. */ - /* The assertions must come before BRA, CBRA, ONCE, and COND, and the four - asserts must remain in order. */ + /* The assertions must come before BRA, CBRA, ONCE, and COND. */ OP_REVERSE, /* 125 Move pointer back - used in lookbehind assertions */ OP_ASSERT, /* 126 Positive lookahead */ OP_ASSERT_NOT, /* 127 Negative lookahead */ OP_ASSERTBACK, /* 128 Positive lookbehind */ OP_ASSERTBACK_NOT, /* 129 Negative lookbehind */ + OP_ASSERT_NA, /* 130 Positive non-atomic lookahead */ + OP_ASSERTBACK_NA, /* 131 Positive non-atomic lookbehind */ /* ONCE, SCRIPT_RUN, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come immediately after the assertions, with ONCE first, as there's a test for >= ONCE for a subpattern that isn't an assertion. The POS versions must immediately follow the non-POS versions in each case. */ - OP_ONCE, /* 130 Atomic group, contains captures */ - OP_SCRIPT_RUN, /* 131 Non-capture, but check characters' scripts */ - OP_BRA, /* 132 Start of non-capturing bracket */ - OP_BRAPOS, /* 133 Ditto, with unlimited, possessive repeat */ - OP_CBRA, /* 134 Start of capturing bracket */ - OP_CBRAPOS, /* 135 Ditto, with unlimited, possessive repeat */ - OP_COND, /* 136 Conditional group */ + OP_ONCE, /* 132 Atomic group, contains captures */ + OP_SCRIPT_RUN, /* 133 Non-capture, but check characters' scripts */ + OP_BRA, /* 134 Start of non-capturing bracket */ + OP_BRAPOS, /* 135 Ditto, with unlimited, possessive repeat */ + OP_CBRA, /* 136 Start of capturing bracket */ + OP_CBRAPOS, /* 137 Ditto, with unlimited, possessive repeat */ + OP_COND, /* 138 Conditional group */ /* These five must follow the previous five, in the same order. There's a check for >= SBRA to distinguish the two sets. */ - OP_SBRA, /* 137 Start of non-capturing bracket, check empty */ - OP_SBRAPOS, /* 138 Ditto, with unlimited, possessive repeat */ - OP_SCBRA, /* 139 Start of capturing bracket, check empty */ - OP_SCBRAPOS, /* 140 Ditto, with unlimited, possessive repeat */ - OP_SCOND, /* 141 Conditional group, check empty */ + OP_SBRA, /* 139 Start of non-capturing bracket, check empty */ + OP_SBRAPOS, /* 149 Ditto, with unlimited, possessive repeat */ + OP_SCBRA, /* 141 Start of capturing bracket, check empty */ + OP_SCBRAPOS, /* 142 Ditto, with unlimited, possessive repeat */ + OP_SCOND, /* 143 Conditional group, check empty */ /* The next two pairs must (respectively) be kept together. */ - OP_CREF, /* 142 Used to hold a capture number as condition */ - OP_DNCREF, /* 143 Used to point to duplicate names as a condition */ - OP_RREF, /* 144 Used to hold a recursion number as condition */ - OP_DNRREF, /* 145 Used to point to duplicate names as a condition */ - OP_FALSE, /* 146 Always false (used by DEFINE and VERSION) */ - OP_TRUE, /* 147 Always true (used by VERSION) */ + OP_CREF, /* 144 Used to hold a capture number as condition */ + OP_DNCREF, /* 145 Used to point to duplicate names as a condition */ + OP_RREF, /* 146 Used to hold a recursion number as condition */ + OP_DNRREF, /* 147 Used to point to duplicate names as a condition */ + OP_FALSE, /* 148 Always false (used by DEFINE and VERSION) */ + OP_TRUE, /* 149 Always true (used by VERSION) */ - OP_BRAZERO, /* 148 These two must remain together and in this */ - OP_BRAMINZERO, /* 149 order. */ - OP_BRAPOSZERO, /* 150 */ + OP_BRAZERO, /* 150 These two must remain together and in this */ + OP_BRAMINZERO, /* 151 order. */ + OP_BRAPOSZERO, /* 152 */ /* These are backtracking control verbs */ - OP_MARK, /* 151 always has an argument */ - OP_PRUNE, /* 152 */ - OP_PRUNE_ARG, /* 153 same, but with argument */ - OP_SKIP, /* 154 */ - OP_SKIP_ARG, /* 155 same, but with argument */ - OP_THEN, /* 156 */ - OP_THEN_ARG, /* 157 same, but with argument */ - OP_COMMIT, /* 158 */ - OP_COMMIT_ARG, /* 159 same, but with argument */ + OP_MARK, /* 153 always has an argument */ + OP_PRUNE, /* 154 */ + OP_PRUNE_ARG, /* 155 same, but with argument */ + OP_SKIP, /* 156 */ + OP_SKIP_ARG, /* 157 same, but with argument */ + OP_THEN, /* 158 */ + OP_THEN_ARG, /* 159 same, but with argument */ + OP_COMMIT, /* 160 */ + OP_COMMIT_ARG, /* 161 same, but with argument */ /* These are forced failure and success verbs. FAIL and ACCEPT do accept an argument, but these cases can be compiled as, for example, (*MARK:X)(*FAIL) without the need for a special opcode. */ - OP_FAIL, /* 160 */ - OP_ACCEPT, /* 161 */ - OP_ASSERT_ACCEPT, /* 162 Used inside assertions */ - OP_CLOSE, /* 163 Used before OP_ACCEPT to close open captures */ + OP_FAIL, /* 162 */ + OP_ACCEPT, /* 163 */ + OP_ASSERT_ACCEPT, /* 164 Used inside assertions */ + OP_CLOSE, /* 165 Used before OP_ACCEPT to close open captures */ /* This is used to skip a subpattern with a {0} quantifier */ - OP_SKIPZERO, /* 164 */ + OP_SKIPZERO, /* 166 */ /* This is used to identify a DEFINE group during compilation so that it can be checked for having only one branch. It is changed to OP_FALSE before compilation finishes. */ - OP_DEFINE, /* 165 */ + OP_DEFINE, /* 167 */ /* This is not an opcode, but is used to check that tables indexed by opcode are the correct length, in order to catch updating errors - there have been @@ -1596,7 +1607,7 @@ enum { /* *** NOTE NOTE NOTE *** Whenever the list above is updated, the two macro definitions that follow must also be updated to match. There are also tables called "opcode_possessify" in pcre2_compile.c and "coptable" and "poptable" in -pcre2_dfa_exec.c that must be updated. */ +pcre2_dfa_match.c that must be updated. */ /* This macro defines textual names for all the opcodes. These are used only @@ -1629,7 +1640,9 @@ some cases doesn't actually use these names at all). */ "class", "nclass", "xclass", "Ref", "Refi", "DnRef", "DnRefi", \ "Recurse", "Callout", "CalloutStr", \ "Alt", "Ket", "KetRmax", "KetRmin", "KetRpos", \ - "Reverse", "Assert", "Assert not", "AssertB", "AssertB not", \ + "Reverse", "Assert", "Assert not", \ + "Assert back", "Assert back not", \ + "Non-atomic assert", "Non-atomic assert back", \ "Once", \ "Script run", \ "Bra", "BraPos", "CBra", "CBraPos", \ @@ -1714,6 +1727,8 @@ in UTF-8 mode. The code that uses this table must know about such things. */ 1+LINK_SIZE, /* Assert not */ \ 1+LINK_SIZE, /* Assert behind */ \ 1+LINK_SIZE, /* Assert behind not */ \ + 1+LINK_SIZE, /* NA Assert */ \ + 1+LINK_SIZE, /* NA Assert behind */ \ 1+LINK_SIZE, /* ONCE */ \ 1+LINK_SIZE, /* SCRIPT_RUN */ \ 1+LINK_SIZE, /* BRA */ \ @@ -1755,13 +1770,11 @@ typedef struct pcre2_memctl { /* Structure for building a chain of open capturing subpatterns during compiling, so that instructions to close them can be compiled when (*ACCEPT) is -encountered. This is also used to identify subpatterns that contain recursive -back references to themselves, so that they can be made atomic. */ +encountered. */ typedef struct open_capitem { struct open_capitem *next; /* Chain link */ uint16_t number; /* Capture number */ - uint16_t flag; /* Set TRUE if recursive back ref */ uint16_t assert_depth; /* Assertion depth when opened */ } open_capitem; @@ -1950,7 +1963,7 @@ is available. */ #define _pcre2_was_newline PCRE2_SUFFIX(_pcre2_was_newline_) #define _pcre2_xclass PCRE2_SUFFIX(_pcre2_xclass_) -extern int _pcre2_auto_possessify(PCRE2_UCHAR *, BOOL, +extern int _pcre2_auto_possessify(PCRE2_UCHAR *, const compile_block *); extern int _pcre2_check_escape(PCRE2_SPTR *, PCRE2_SPTR, uint32_t *, int *, uint32_t, uint32_t, BOOL, compile_block *); diff --git a/dist2/src/pcre2_intmodedep.h b/dist2/src/pcre2_intmodedep.h index bf3a2359..ea3b3ec6 100644 --- a/dist2/src/pcre2_intmodedep.h +++ b/dist2/src/pcre2_intmodedep.h @@ -205,19 +205,19 @@ whether its argument, which is assumed to be one code unit, is less than 256. The CHMAX_255 macro does not assume one code unit. The maximum length of a MARK name must fit in one code unit; currently it is set to 255 or 65535. The TABLE_GET macro is used to access elements of tables containing exactly 256 -items. When code points can be greater than 255, a check is needed before -accessing these tables. */ +items. Its argument is a code unit. When code points can be greater than 255, a +check is needed before accessing these tables. */ #if PCRE2_CODE_UNIT_WIDTH == 8 #define MAX_255(c) TRUE #define MAX_MARK ((1u << 8) - 1) +#define TABLE_GET(c, table, default) ((table)[c]) #ifdef SUPPORT_UNICODE #define SUPPORT_WIDE_CHARS #define CHMAX_255(c) ((c) <= 255u) #else #define CHMAX_255(c) TRUE #endif /* SUPPORT_UNICODE */ -#define TABLE_GET(c, table, default) ((table)[c]) #else /* Code units are 16 or 32 bits */ #define CHMAX_255(c) ((c) <= 255u) @@ -228,7 +228,6 @@ accessing these tables. */ #endif - /* ----------------- Character-handling macros ----------------- */ /* There is a proposed future special "UTF-21" mode, in which only the lowest @@ -854,6 +853,7 @@ typedef struct match_block { uint32_t match_call_count; /* Number of times a new frame is created */ BOOL hitend; /* Hit the end of the subject at some point */ BOOL hasthen; /* Pattern contains (*THEN) */ + BOOL allowemptypartial; /* Allow empty hard partial */ const uint8_t *lcc; /* Points to lower casing table */ const uint8_t *fcc; /* Points to case-flipping table */ const uint8_t *ctypes; /* Points to table of type maps */ @@ -866,6 +866,7 @@ typedef struct match_block { PCRE2_SPTR name_table; /* Table of group names */ PCRE2_SPTR start_code; /* For use when recursing */ PCRE2_SPTR start_subject; /* Start of the subject string */ + PCRE2_SPTR check_subject; /* Where UTF-checked from */ PCRE2_SPTR end_subject; /* End of the subject string */ PCRE2_SPTR end_match_ptr; /* Subject position at end match */ PCRE2_SPTR start_used_ptr; /* Earliest consulted character */ @@ -908,6 +909,7 @@ typedef struct dfa_match_block { uint32_t poptions; /* Pattern options */ uint32_t nltype; /* Newline type */ uint32_t nllen; /* Newline string length */ + BOOL allowemptypartial; /* Allow empty hard partial */ PCRE2_UCHAR nl[4]; /* Newline string when fixed */ uint16_t bsr_convention; /* \R interpretation */ pcre2_callout_block *cb; /* Points to a callout block */ diff --git a/dist2/src/pcre2_jit_compile.c b/dist2/src/pcre2_jit_compile.c index 1f21bfb6..1977d28a 100644 --- a/dist2/src/pcre2_jit_compile.c +++ b/dist2/src/pcre2_jit_compile.c @@ -6,8 +6,9 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel + This module by Zoltan Herczeg Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -212,12 +213,6 @@ typedef struct stub_list { struct stub_list *next; } stub_list; -typedef struct label_addr_list { - struct sljit_label *label; - sljit_uw *update_addr; - struct label_addr_list *next; -} label_addr_list; - enum frame_types { no_frame = -1, no_stack = -2 @@ -228,6 +223,12 @@ enum control_types { type_then_trap = 1 }; +enum early_fail_types { + type_skip = 0, + type_fail = 1, + type_fail_range = 2 +}; + typedef int (SLJIT_FUNC *jit_function)(jit_arguments *args); /* The following structure is the key data type for the recursive @@ -271,6 +272,8 @@ typedef struct bracket_backtrack { assert_backtrack *assert; /* For OP_ONCE. Less than 0 if not needed. */ int framesize; + /* For brackets with >3 alternatives. */ + struct sljit_put_label *matching_put_label; } u; /* Points to our private memory word on the stack. */ int private_data_ptr; @@ -408,14 +411,16 @@ typedef struct compiler_common { /* Fast forward skipping byte code pointer. */ PCRE2_SPTR fast_forward_bc_ptr; /* Locals used by fast fail optimization. */ - sljit_s32 fast_fail_start_ptr; - sljit_s32 fast_fail_end_ptr; + sljit_s32 early_fail_start_ptr; + sljit_s32 early_fail_end_ptr; /* Flipped and lower case tables. */ const sljit_u8 *fcc; sljit_sw lcc; /* Mode can be PCRE2_JIT_COMPLETE and others. */ int mode; + /* TRUE, when empty match is accepted for partial matching. */ + BOOL allow_empty_partial; /* TRUE, when minlength is greater than 0. */ BOOL might_be_empty; /* \K is found in the pattern. */ @@ -454,7 +459,6 @@ typedef struct compiler_common { struct sljit_label *accept_label; struct sljit_label *ff_newline_shortcut; stub_list *stubs; - label_addr_list *label_addrs; recurse_entry *entries; recurse_entry *currententry; jump_list *partialmatch; @@ -478,7 +482,7 @@ typedef struct compiler_common { #ifdef SUPPORT_UNICODE BOOL utf; BOOL invalid_utf; - BOOL use_ucp; + BOOL ucp; /* Points to saving area for iref. */ sljit_s32 iref_ptr; jump_list *getucd; @@ -563,6 +567,12 @@ typedef struct compare_context { #define ARGUMENTS SLJIT_S4 #define RETURN_ADDR SLJIT_R4 +#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) +#define HAS_VIRTUAL_REGISTERS 1 +#else +#define HAS_VIRTUAL_REGISTERS 0 +#endif + /* Local space layout. */ /* These two locals can be used by the current opcode. */ #define LOCALS0 (0 * sizeof(sljit_sw)) @@ -603,6 +613,8 @@ the start pointers when the end of the capturing group has not yet reached. */ sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw)) #define OP2(op, dst, dstw, src1, src1w, src2, src2w) \ sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w)) +#define OP_SRC(op, src, srcw) \ + sljit_emit_op_src(compiler, (op), (src), (srcw)) #define LABEL() \ sljit_emit_label(compiler) #define JUMP(type) \ @@ -696,11 +708,12 @@ the start pointers when the end of the capturing group has not yet reached. */ #define GETCHARBACK_INVALID(c, ptr, start, invalid_action) \ { \ - if (ptr[-1] <= 0x7f) \ - c = *ptr--; \ + c = ptr[-1]; \ + if (c <= 0x7f) \ + ptr--; \ else if (ptr - 1 > start && ptr[-1] >= 0x80 && ptr[-1] < 0xc0) \ { \ - c = ptr[-1] - 0x80; \ + c -= 0x80; \ \ if (ptr[-2] >= 0xc2 && ptr[-2] <= 0xdf) \ { \ @@ -775,11 +788,12 @@ the start pointers when the end of the capturing group has not yet reached. */ #define GETCHARBACK_INVALID(c, ptr, start, invalid_action) \ { \ - if (ptr[-1] < 0xd800 || ptr[-1] >= 0xe000) \ - c = *ptr--; \ - else if (ptr[-1] >= 0xdc00 && ptr - 1 > start && ptr[-2] >= 0xd800 && ptr[-2] < 0xdc00) \ + c = ptr[-1]; \ + if (c < 0xd800 || c >= 0xe000) \ + ptr--; \ + else if (c >= 0xdc00 && ptr - 1 > start && ptr[-2] >= 0xd800 && ptr[-2] < 0xdc00) \ { \ - c = (((ptr[-2] - 0xd800) << 10) | (ptr[-1] - 0xdc00)) + 0x10000; \ + c = (((ptr[-2] - 0xd800) << 10) | (c - 0xdc00)) + 0x10000; \ ptr -= 2; \ } \ else \ @@ -793,7 +807,7 @@ the start pointers when the end of the capturing group has not yet reached. */ #define GETCHARINC_INVALID(c, ptr, end, invalid_action) \ { \ - if (ptr[0] < 0x110000) \ + if (ptr[0] < 0xd800 || (ptr[0] >= 0xe000 && ptr[0] < 0x110000)) \ c = *ptr++; \ else \ { \ @@ -801,12 +815,23 @@ the start pointers when the end of the capturing group has not yet reached. */ } \ } +#define GETCHARBACK_INVALID(c, ptr, start, invalid_action) \ + { \ + c = ptr[-1]; \ + if (ptr[-1] < 0xd800 || (ptr[-1] >= 0xe000 && ptr[-1] < 0x110000)) \ + ptr--; \ + else \ + { \ + invalid_action; \ + } \ + } + #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ #endif /* SUPPORT_UNICODE */ static PCRE2_SPTR bracketend(PCRE2_SPTR cc) { -SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); +SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NA) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); do cc += GET(cc, 1); while (*cc == OP_ALT); SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS); cc += 1 + LINK_SIZE; @@ -816,7 +841,7 @@ return cc; static int no_alternatives(PCRE2_SPTR cc) { int count = 0; -SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); +SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NA) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); do { cc += GET(cc, 1); @@ -901,6 +926,8 @@ switch(*cc) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRA: @@ -1033,7 +1060,6 @@ switch(*cc) return cc + 1 + 2 + cc[1]; default: - /* All opcodes are supported now! */ SLJIT_UNREACHABLE(); return NULL; } @@ -1044,6 +1070,7 @@ static BOOL check_opcode_types(compiler_common *common, PCRE2_SPTR cc, PCRE2_SPT int count; PCRE2_SPTR slot; PCRE2_SPTR assert_back_end = cc - 1; +PCRE2_SPTR assert_na_end = cc - 1; /* Calculate important variables (like stack size) and checks whether all opcodes are supported. */ while (cc < ccend) @@ -1070,6 +1097,14 @@ while (cc < ccend) cc += 1 + IMM2_SIZE; break; + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: + slot = bracketend(cc); + if (slot > assert_na_end) + assert_na_end = slot; + cc += 1 + LINK_SIZE; + break; + case OP_CBRAPOS: case OP_SCBRAPOS: common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] = 0; @@ -1137,6 +1172,9 @@ while (cc < ccend) case OP_COMMIT_ARG: case OP_PRUNE_ARG: + if (cc < assert_na_end) + return FALSE; + /* Fall through */ case OP_MARK: if (common->mark_ptr == 0) { @@ -1155,6 +1193,8 @@ while (cc < ccend) case OP_SKIP: if (cc < assert_back_end) common->has_skip_in_assert_back = TRUE; + if (cc < assert_na_end) + return FALSE; cc += 1; break; @@ -1163,9 +1203,19 @@ while (cc < ccend) common->has_skip_arg = TRUE; if (cc < assert_back_end) common->has_skip_in_assert_back = TRUE; + if (cc < assert_na_end) + return FALSE; cc += 1 + 2 + cc[1]; break; + case OP_PRUNE: + case OP_COMMIT: + case OP_ASSERT_ACCEPT: + if (cc < assert_na_end) + return FALSE; + cc++; + break; + default: cc = next_opcode(common, cc); if (cc == NULL) @@ -1176,183 +1226,355 @@ while (cc < ccend) return TRUE; } -static BOOL is_accelerated_repeat(PCRE2_SPTR cc) +#define EARLY_FAIL_ENHANCE_MAX (1 + 1) + +/* +start: + 0 - skip / early fail allowed + 1 - only early fail with range allowed + >1 - (start - 1) early fail is processed + +return: current number of iterators enhanced with fast fail +*/ +static int detect_early_fail(compiler_common *common, PCRE2_SPTR cc, int *private_data_start, sljit_s32 depth, int start) { -switch(*cc) - { - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - return (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI); +PCRE2_SPTR next_alt; +PCRE2_SPTR end; +PCRE2_SPTR accelerated_start; +int result = 0; +int count; +BOOL fast_forward_allowed = TRUE; - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_POSSTAR: - case OP_POSPLUS: +SLJIT_ASSERT(*cc == OP_ONCE || *cc == OP_BRA || *cc == OP_CBRA); +SLJIT_ASSERT(*cc != OP_CBRA || common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] != 0); +SLJIT_ASSERT(start < EARLY_FAIL_ENHANCE_MAX); - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSSTARI: - case OP_POSPLUSI: +do + { + count = start; + next_alt = cc + GET(cc, 1); + cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0); - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: + while (TRUE) + { + accelerated_start = NULL; - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - return TRUE; + switch(*cc) + { + case OP_SOD: + case OP_SOM: + case OP_SET_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + /* Zero width assertions. */ + cc++; + continue; + + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_ANYBYTE: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + fast_forward_allowed = FALSE; + cc++; + continue; - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8 - case OP_XCLASS: - cc += (*cc == OP_XCLASS) ? GET(cc, 1) : (int)(1 + (32 / sizeof(PCRE2_UCHAR))); -#else - cc += (1 + (32 / sizeof(PCRE2_UCHAR))); + case OP_ANYNL: + case OP_EXTUNI: + fast_forward_allowed = FALSE; + if (count == 0) + count = 1; + cc++; + continue; + + case OP_NOTPROP: + case OP_PROP: + fast_forward_allowed = FALSE; + cc += 1 + 2; + continue; + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + fast_forward_allowed = FALSE; + cc += 2; +#ifdef SUPPORT_UNICODE + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); #endif + continue; - switch(*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - return TRUE; - } - break; - } -return FALSE; -} + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + /* The type or prop opcode is skipped in the next iteration. */ + cc += 1; -static SLJIT_INLINE BOOL detect_fast_forward_skip(compiler_common *common, int *private_data_start) -{ -PCRE2_SPTR cc = common->start; -PCRE2_SPTR end; + if (cc[0] != OP_ANYNL && cc[0] != OP_EXTUNI) + { + accelerated_start = cc - 1; + break; + } -/* Skip not repeated brackets. */ -while (TRUE) - { - switch(*cc) - { - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - /* Zero width assertions. */ - cc++; - continue; - } + if (count == 0) + count = 1; + fast_forward_allowed = FALSE; + continue; - if (*cc != OP_BRA && *cc != OP_CBRA) - break; + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSUPTO: + cc += IMM2_SIZE; + /* Fall through */ + + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSQUERY: + /* The type or prop opcode is skipped in the next iteration. */ + fast_forward_allowed = FALSE; + if (count == 0) + count = 1; + cc += 1; + continue; + + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_POSSTAR: + case OP_POSPLUS: + + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSSTARI: + case OP_POSPLUSI: + + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + accelerated_start = cc; + cc += 2; +#ifdef SUPPORT_UNICODE + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; - end = cc + GET(cc, 1); - if (*end != OP_KET || PRIVATE_DATA(end) != 0) - return FALSE; - if (*cc == OP_CBRA) - { - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - return FALSE; - cc += IMM2_SIZE; - } - cc += 1 + LINK_SIZE; - } + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSUPTO: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSUPTOI: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSUPTO: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSUPTOI: + cc += IMM2_SIZE; + /* Fall through */ + + case OP_QUERY: + case OP_MINQUERY: + case OP_POSQUERY: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_POSQUERYI: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTPOSQUERY: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTPOSQUERYI: + fast_forward_allowed = FALSE; + if (count == 0) + count = 1; + cc += 2; +#ifdef SUPPORT_UNICODE + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + continue; -if (is_accelerated_repeat(cc)) - { - common->fast_forward_bc_ptr = cc; - common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; - *private_data_start += sizeof(sljit_sw); - return TRUE; - } -return FALSE; -} + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8 + case OP_XCLASS: + accelerated_start = cc; + cc += ((*cc == OP_XCLASS) ? GET(cc, 1) : (unsigned int)(1 + (32 / sizeof(PCRE2_UCHAR)))); +#else + accelerated_start = cc; + cc += (1 + (32 / sizeof(PCRE2_UCHAR))); +#endif -static SLJIT_INLINE void detect_fast_fail(compiler_common *common, PCRE2_SPTR cc, int *private_data_start, sljit_s32 depth) -{ - PCRE2_SPTR next_alt; + switch (*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + cc++; + break; - SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA); + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + cc += 2 * IMM2_SIZE; + /* Fall through */ + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSQUERY: + cc++; + if (count == 0) + count = 1; + /* Fall through */ + default: + accelerated_start = NULL; + fast_forward_allowed = FALSE; + continue; + } + break; - if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - return; + case OP_ONCE: + case OP_BRA: + case OP_CBRA: + end = cc + GET(cc, 1); - next_alt = bracketend(cc) - (1 + LINK_SIZE); - if (*next_alt != OP_KET || PRIVATE_DATA(next_alt) != 0) - return; + if (*end == OP_KET && PRIVATE_DATA(end) == 0) + { + if (*cc == OP_CBRA) + { + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + break; + cc += IMM2_SIZE; + } - do - { - next_alt = cc + GET(cc, 1); + cc += 1 + LINK_SIZE; + continue; + } - cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0); + fast_forward_allowed = FALSE; + if (depth >= 4) + break; - while (TRUE) - { - switch(*cc) + end = bracketend(cc) - (1 + LINK_SIZE); + if (*end != OP_KET || PRIVATE_DATA(end) != 0) + break; + + if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + break; + + count = detect_early_fail(common, cc, private_data_start, depth + 1, count); + if (count < EARLY_FAIL_ENHANCE_MAX) { - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - /* Zero width assertions. */ - cc++; + cc = end + (1 + LINK_SIZE); continue; } break; - } - if (depth > 0 && (*cc == OP_BRA || *cc == OP_CBRA)) - detect_fast_fail(common, cc, private_data_start, depth - 1); + case OP_KET: + SLJIT_ASSERT(PRIVATE_DATA(cc) == 0); + if (cc >= next_alt) + break; + cc += 1 + LINK_SIZE; + continue; + } - if (is_accelerated_repeat(cc)) + if (accelerated_start != NULL) { - common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; + if (count == 0) + { + count++; + + if (fast_forward_allowed && *next_alt == OP_KET) + { + common->fast_forward_bc_ptr = accelerated_start; + common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_skip; + *private_data_start += sizeof(sljit_sw); + } + else + { + common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail; - if (common->fast_fail_start_ptr == 0) - common->fast_fail_start_ptr = *private_data_start; + if (common->early_fail_start_ptr == 0) + common->early_fail_start_ptr = *private_data_start; - *private_data_start += sizeof(sljit_sw); - common->fast_fail_end_ptr = *private_data_start; + *private_data_start += sizeof(sljit_sw); + common->early_fail_end_ptr = *private_data_start; - if (*private_data_start > SLJIT_MAX_LOCAL_SIZE) - return; + if (*private_data_start > SLJIT_MAX_LOCAL_SIZE) + return EARLY_FAIL_ENHANCE_MAX; + } + } + else + { + common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail_range; + + if (common->early_fail_start_ptr == 0) + common->early_fail_start_ptr = *private_data_start; + + *private_data_start += 2 * sizeof(sljit_sw); + common->early_fail_end_ptr = *private_data_start; + + if (*private_data_start > SLJIT_MAX_LOCAL_SIZE) + return EARLY_FAIL_ENHANCE_MAX; + } + + count++; + + if (count < EARLY_FAIL_ENHANCE_MAX) + continue; } - cc = next_alt; + break; } - while (*cc == OP_ALT); + + if (*cc != OP_ALT && *cc != OP_KET) + result = EARLY_FAIL_ENHANCE_MAX; + else if (result < count) + result = count; + + fast_forward_allowed = FALSE; + cc = next_alt; + } +while (*cc == OP_ALT); + +return result; } static int get_class_iterator_size(PCRE2_SPTR cc) @@ -1569,6 +1791,8 @@ while (cc < ccend) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRAPOS: @@ -1610,57 +1834,57 @@ while (cc < ccend) case OP_BRAZERO: case OP_BRAMINZERO: case OP_BRAPOSZERO: - repeat_check = FALSE; size = 1; + repeat_check = FALSE; break; CASE_ITERATOR_PRIVATE_DATA_1 - space = 1; size = -2; + space = 1; break; CASE_ITERATOR_PRIVATE_DATA_2A - space = 2; size = -2; + space = 2; break; CASE_ITERATOR_PRIVATE_DATA_2B - space = 2; size = -(2 + IMM2_SIZE); + space = 2; break; CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - space = 1; size = 1; + space = 1; break; CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + size = 1; if (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI) space = 2; - size = 1; break; case OP_TYPEUPTO: + size = 1 + IMM2_SIZE; if (cc[1 + IMM2_SIZE] != OP_ANYNL && cc[1 + IMM2_SIZE] != OP_EXTUNI) space = 2; - size = 1 + IMM2_SIZE; break; case OP_TYPEMINUPTO: - space = 2; size = 1 + IMM2_SIZE; + space = 2; break; case OP_CLASS: case OP_NCLASS: - space = get_class_iterator_size(cc + size); size = 1 + 32 / sizeof(PCRE2_UCHAR); + space = get_class_iterator_size(cc + size); break; #if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8 case OP_XCLASS: - space = get_class_iterator_size(cc + size); size = GET(cc, 1); + space = get_class_iterator_size(cc + size); break; #endif @@ -2146,6 +2370,8 @@ while (cc < ccend) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRAPOS: @@ -2371,14 +2597,14 @@ if (base_reg != TMP2) else { status.saved_tmp_regs[1] = RETURN_ADDR; - if (sljit_get_register_index(RETURN_ADDR) == -1) + if (HAS_VIRTUAL_REGISTERS) status.tmp_regs[1] = STR_PTR; else status.tmp_regs[1] = RETURN_ADDR; } status.saved_tmp_regs[2] = TMP3; -if (sljit_get_register_index(TMP3) == -1) +if (HAS_VIRTUAL_REGISTERS) status.tmp_regs[2] = STR_END; else status.tmp_regs[2] = TMP3; @@ -2470,6 +2696,8 @@ while (cc < ccend) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRAPOS: @@ -2643,8 +2871,8 @@ while (cc < ccend) } if (common->control_head_ptr != 0 && !control_head_found) { - shared_srcw[0] = common->control_head_ptr; - shared_count = 1; + private_srcw[0] = common->control_head_ptr; + private_count = 1; control_head_found = TRUE; } cc += 1 + 2 + cc[1]; @@ -2654,8 +2882,8 @@ while (cc < ccend) SLJIT_ASSERT(common->control_head_ptr != 0); if (!control_head_found) { - shared_srcw[0] = common->control_head_ptr; - shared_count = 1; + private_srcw[0] = common->control_head_ptr; + private_count = 1; control_head_found = TRUE; } cc++; @@ -2739,7 +2967,7 @@ PCRE2_SPTR end = bracketend(cc); BOOL has_alternatives = cc[GET(cc, 1)] == OP_ALT; /* Assert captures then. */ -if (*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) +if (*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NA) current_offset = NULL; /* Conditional block does not. */ if (*cc == OP_COND || *cc == OP_SCOND) @@ -2751,7 +2979,7 @@ if (has_alternatives) while (cc < end) { - if ((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)) + if ((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NA) || (*cc >= OP_ONCE && *cc <= OP_SCOND)) cc = set_then_offsets(common, cc, current_offset); else { @@ -2829,20 +3057,6 @@ while (list_item) common->stubs = NULL; } -static void add_label_addr(compiler_common *common, sljit_uw *update_addr) -{ -DEFINE_COMPILER; -label_addr_list *label_addr; - -label_addr = sljit_alloc_memory(compiler, sizeof(label_addr_list)); -if (label_addr == NULL) - return; -label_addr->label = LABEL(); -label_addr->update_addr = update_addr; -label_addr->next = common->label_addrs; -common->label_addrs = label_addr; -} - static SLJIT_INLINE void count_match(compiler_common *common) { DEFINE_COMPILER; @@ -2935,16 +3149,54 @@ else } } -static SLJIT_INLINE void reset_fast_fail(compiler_common *common) +static SLJIT_INLINE void reset_early_fail(compiler_common *common) { DEFINE_COMPILER; +sljit_u32 size = (sljit_u32)(common->early_fail_end_ptr - common->early_fail_start_ptr); +sljit_u32 uncleared_size; +sljit_s32 src = SLJIT_IMM; sljit_s32 i; +struct sljit_label *loop; -SLJIT_ASSERT(common->fast_fail_start_ptr < common->fast_fail_end_ptr); +SLJIT_ASSERT(common->early_fail_start_ptr < common->early_fail_end_ptr); -OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -for (i = common->fast_fail_start_ptr; i < common->fast_fail_end_ptr; i += sizeof(sljit_sw)) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, TMP1, 0); +if (size == sizeof(sljit_sw)) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->early_fail_start_ptr, SLJIT_IMM, 0); + return; + } + +if (sljit_get_register_index(TMP3) >= 0 && !sljit_has_cpu_feature(SLJIT_HAS_ZERO_REGISTER)) + { + OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); + src = TMP3; + } + +if (size <= 6 * sizeof(sljit_sw)) + { + for (i = common->early_fail_start_ptr; i < common->early_fail_end_ptr; i += sizeof(sljit_sw)) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, src, 0); + return; + } + +GET_LOCAL_BASE(TMP1, 0, common->early_fail_start_ptr); + +uncleared_size = ((size / sizeof(sljit_sw)) % 3) * sizeof(sljit_sw); + +OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, size - uncleared_size); + +loop = LABEL(); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), -2 * (sljit_sw)sizeof(sljit_sw), src, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), -1 * (sljit_sw)sizeof(sljit_sw), src, 0); +CMPTO(SLJIT_LESS, TMP1, 0, TMP2, 0, loop); + +if (uncleared_size >= sizeof(sljit_sw)) + OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0); + +if (uncleared_size >= 2 * sizeof(sljit_sw)) + OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), sizeof(sljit_sw), src, 0); } static SLJIT_INLINE void do_reset_match(compiler_common *common, int length) @@ -2985,12 +3237,18 @@ else } } -OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0); +if (!HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, stack)); +else + OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0); + if (common->mark_ptr != 0) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0); if (common->control_head_ptr != 0) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack)); +if (HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, end)); } @@ -3029,21 +3287,36 @@ BOOL has_pre; OP1(SLJIT_MOV, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(1), STR_PTR, 0); -OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); -OP1(SLJIT_MOV_U32, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, oveccount)); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, startchar_ptr), SLJIT_S0, 0); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0); -OP2(SLJIT_ADD, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, match_data), - SLJIT_IMM, SLJIT_OFFSETOF(pcre2_match_data, ovector) - sizeof(PCRE2_SIZE)); +if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); + if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + OP1(SLJIT_MOV_U32, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, oveccount)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, startchar_ptr), SLJIT_S0, 0); + if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0); + OP2(SLJIT_ADD, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, match_data), + SLJIT_IMM, SLJIT_OFFSETOF(pcre2_match_data, ovector) - sizeof(PCRE2_SIZE)); + } +else + { + OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); + OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, match_data)); + if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + OP1(SLJIT_MOV_U32, SLJIT_R1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, oveccount)); + OP1(SLJIT_MOV, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, startchar_ptr), SLJIT_S0, 0); + if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R0, 0); + OP2(SLJIT_ADD, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, SLJIT_OFFSETOF(pcre2_match_data, ovector) - sizeof(PCRE2_SIZE)); + } has_pre = sljit_emit_mem(compiler, SLJIT_MOV | SLJIT_MEM_SUPP | SLJIT_MEM_PRE, SLJIT_S1, SLJIT_MEM1(SLJIT_S0), sizeof(sljit_sw)) == SLJIT_SUCCESS; GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START - (has_pre ? sizeof(sljit_sw) : 0)); -OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, begin)); +OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(HAS_VIRTUAL_REGISTERS ? SLJIT_R0 : ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); loop = LABEL(); @@ -3105,20 +3378,22 @@ static SLJIT_INLINE void return_with_partial_match(compiler_common *common, stru { DEFINE_COMPILER; sljit_s32 mov_opcode; +sljit_s32 arguments_reg = !HAS_VIRTUAL_REGISTERS ? ARGUMENTS : SLJIT_R1; SLJIT_COMPILE_ASSERT(STR_END == SLJIT_S0, str_end_must_be_saved_reg0); SLJIT_ASSERT(common->start_used_ptr != 0 && common->start_ptr != 0 && (common->mode == PCRE2_JIT_PARTIAL_SOFT ? common->hit_start != 0 : common->hit_start == 0)); -OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0); +if (arguments_reg != ARGUMENTS) + OP1(SLJIT_MOV, arguments_reg, 0, ARGUMENTS, 0); OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == PCRE2_JIT_PARTIAL_SOFT ? common->hit_start : common->start_ptr); OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE2_ERROR_PARTIAL); /* Store match begin and end. */ -OP1(SLJIT_MOV, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, begin)); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, startchar_ptr), SLJIT_R2, 0); -OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, match_data)); +OP1(SLJIT_MOV, SLJIT_S1, 0, SLJIT_MEM1(arguments_reg), SLJIT_OFFSETOF(jit_arguments, begin)); +OP1(SLJIT_MOV, SLJIT_MEM1(arguments_reg), SLJIT_OFFSETOF(jit_arguments, startchar_ptr), SLJIT_R2, 0); +OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(arguments_reg), SLJIT_OFFSETOF(jit_arguments, match_data)); mov_opcode = (sizeof(PCRE2_SIZE) == 4) ? SLJIT_MOV_U32 : SLJIT_MOV; @@ -3167,16 +3442,19 @@ static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, PCRE2_SPTR unsigned int c; #ifdef SUPPORT_UNICODE -if (common->utf) +if (common->utf || common->ucp) { - GETCHAR(c, cc); - if (c > 127) + if (common->utf) { - return c != UCD_OTHERCASE(c); + GETCHAR(c, cc); } -#if PCRE2_CODE_UNIT_WIDTH != 8 + else + c = *cc; + + if (c > 127) + return c != UCD_OTHERCASE(c); + return common->fcc[c] != c; -#endif } else #endif @@ -3188,10 +3466,8 @@ static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigne { /* Returns with the othercase. */ #ifdef SUPPORT_UNICODE -if (common->utf && c > 127) - { +if ((common->utf || common->ucp) && c > 127) return UCD_OTHERCASE(c); - } #endif return TABLE_GET(c, common->fcc, c); } @@ -3205,15 +3481,19 @@ int n; #endif #ifdef SUPPORT_UNICODE -if (common->utf) +if (common->utf || common->ucp) { - GETCHAR(c, cc); + if (common->utf) + { + GETCHAR(c, cc); + } + else + c = *cc; + if (c <= 127) oc = common->fcc[c]; else - { oc = UCD_OTHERCASE(c); - } } else { @@ -3279,7 +3559,7 @@ SLJIT_ASSERT(!force || common->mode != PCRE2_JIT_COMPLETE); if (common->mode == PCRE2_JIT_COMPLETE) return; -if (!force) +if (!force && !common->allow_empty_partial) jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); else if (common->mode == PCRE2_JIT_PARTIAL_SOFT) jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1); @@ -3341,7 +3621,11 @@ if (common->mode == PCRE2_JIT_COMPLETE) /* Partial matching mode. */ jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); -add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); +if (!common->allow_empty_partial) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); +else if (common->mode == PCRE2_JIT_PARTIAL_SOFT) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1)); + if (common->mode == PCRE2_JIT_PARTIAL_SOFT) { OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); @@ -3357,6 +3641,35 @@ else JUMPHERE(jump); } +static void process_partial_match(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; + +/* Partial matching mode. */ +if (common->mode == PCRE2_JIT_PARTIAL_SOFT) + { + jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); + JUMPHERE(jump); + } +else if (common->mode == PCRE2_JIT_PARTIAL_HARD) + { + if (common->partialmatchlabel != NULL) + CMPTO(SLJIT_LESS, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0, common->partialmatchlabel); + else + add_jump(compiler, &common->partialmatch, CMP(SLJIT_LESS, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); + } +} + +static void detect_partial_match_to(compiler_common *common, struct sljit_label *label) +{ +DEFINE_COMPILER; + +CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, label); +process_partial_match(common); +} + static void peek_char(compiler_common *common, sljit_u32 max, sljit_s32 dst, sljit_sw dstw, jump_list **backtracks) { /* Reads the character into TMP1, keeps STR_PTR. @@ -3420,12 +3733,21 @@ if (common->utf) #elif PCRE2_CODE_UNIT_WIDTH == 32 if (common->invalid_utf) { + if (max < 0xd800) return; + if (backtracks != NULL) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800)); + } else { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x110000); CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); } } #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ @@ -3490,8 +3812,12 @@ if (common->utf) JUMPHERE(jump); } #elif PCRE2_CODE_UNIT_WIDTH == 32 - if (common->invalid_utf) - add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); +if (common->invalid_utf) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800)); + } #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ #endif /* SUPPORT_UNICODE */ } @@ -3653,7 +3979,7 @@ if (common->utf) /* Skip low surrogate if necessary. */ OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && sljit_get_register_index(RETURN_ADDR) >= 0) + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && !HAS_VIRTUAL_REGISTERS) { if (options & READ_CHAR_UPDATE_STR_PTR) OP2(SLJIT_ADD, RETURN_ADDR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -3677,11 +4003,18 @@ if (common->utf) if (common->invalid_utf) { if (backtracks != NULL) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800)); + } else { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x110000); CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); } } #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ @@ -3832,7 +4165,7 @@ if (common->utf && negated) { OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); - if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && sljit_get_register_index(RETURN_ADDR) >= 0) + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && !HAS_VIRTUAL_REGISTERS) { OP2(SLJIT_ADD, RETURN_ADDR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x400); @@ -3865,9 +4198,9 @@ if (common->utf && negated) static void move_back(compiler_common *common, jump_list **backtracks, BOOL must_be_valid) { -/* Goes one character back. TMP2 must contain the start of -the subject buffer. Affects STR_PTR and TMP1. Does not modify -STR_PTR for invalid character sequences. */ +/* Goes one character back. Affects STR_PTR and TMP1. If must_be_valid is TRUE, +TMP2 is not used. Otherwise TMP2 must contain the start of the subject buffer, +and it is destroyed. Does not modify STR_PTR for invalid character sequences. */ DEFINE_COMPILER; SLJIT_UNUSED_ARG(backtracks); @@ -4004,7 +4337,7 @@ jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3000); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); @@ -4017,7 +4350,7 @@ jump = JUMP(SLJIT_NOT_ZERO); /* Three byte sequence. */ OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0000); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Four byte sequence. */ JUMPHERE(jump); @@ -4027,7 +4360,7 @@ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfreadtype8(compiler_common *common) @@ -4052,18 +4385,18 @@ OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(compare); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* We only have types for characters less than 256. */ JUMPHERE(jump); OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfreadchar_invalid(compiler_common *common) @@ -4103,7 +4436,7 @@ OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump); @@ -4146,7 +4479,7 @@ if (has_cmov) } else exit_invalid[4] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump); @@ -4175,7 +4508,7 @@ else exit_invalid[6] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x100000); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(buffer_end_close); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); @@ -4192,7 +4525,7 @@ exit_invalid[8] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Three-byte sequence. */ JUMPHERE(jump); @@ -4222,7 +4555,7 @@ for (i = 0; i < 11; i++) sljit_set_label(exit_invalid[i], exit_invalid_label); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfreadnewline_invalid(compiler_common *common) @@ -4245,7 +4578,14 @@ if (common->nltype != NLTYPE_ANY) /* All newlines are ascii, just skip intermediate octets. */ jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); loop = LABEL(); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_SUPP | SLJIT_MEM_POST, TMP2, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)) == SLJIT_SUCCESS) + sljit_emit_mem(compiler, MOV_UCHAR | SLJIT_MEM_POST, TMP2, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + else + { + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc0); CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0x80, loop); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -4253,7 +4593,7 @@ if (common->nltype != NLTYPE_ANY) JUMPHERE(jump[0]); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); - sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); return; } @@ -4284,14 +4624,14 @@ JUMPHERE(jump[0]); JUMPHERE(jump[4]); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Two byte long newline: 0x85. */ JUMPHERE(jump[1]); CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0x85, skip_start); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x85); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Three byte long newlines: 0x2028 and 0x2029. */ JUMPHERE(jump[2]); @@ -4306,7 +4646,7 @@ CMPTO(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x40, skip_start); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0x2000); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfmoveback_invalid(compiler_common *common) @@ -4335,7 +4675,7 @@ jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x20); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Three-byte sequence. */ JUMPHERE(jump); @@ -4348,7 +4688,7 @@ jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x10); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Four-byte sequence. */ JUMPHERE(jump); @@ -4361,7 +4701,7 @@ exit_invalid[3] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x05); exit_ok_label = LABEL(); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); /* Two-byte sequence. */ JUMPHERE(buffer_start_close); @@ -4391,7 +4731,7 @@ sljit_set_label(exit_invalid[5], exit_invalid_label); sljit_set_label(exit_invalid[6], exit_invalid_label); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(exit_invalid[4]); /* -2 + 4 = 2 */ @@ -4402,12 +4742,12 @@ for (i = 0; i < 4; i++) sljit_set_label(exit_invalid[i], exit_invalid_label); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(4)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfpeakcharback(compiler_common *common) { -/* Peak a character back. */ +/* Peak a character back. Does not modify STR_PTR. */ DEFINE_COMPILER; struct sljit_jump *jump[2]; @@ -4439,12 +4779,12 @@ OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfpeakcharback_invalid(compiler_common *common) { -/* Peak a character back. */ +/* Peak a character back. Does not modify STR_PTR. */ DEFINE_COMPILER; sljit_s32 i; sljit_s32 has_cmov = sljit_has_cpu_feature(SLJIT_HAS_CMOV); @@ -4469,7 +4809,7 @@ two_byte_entry = LABEL(); OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); /* If TMP1 is in 0x80-0xbf range, TMP1 is also increased by (0x2 << 6). */ OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump[1]); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2 - 0x80); @@ -4507,7 +4847,7 @@ if (has_cmov) else exit_invalid[3] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump[1]); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xe0 - 0x80); @@ -4533,7 +4873,7 @@ else exit_invalid[5] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x100000); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump[0]); OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -4556,7 +4896,7 @@ OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xe0); CMPTO(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x10, three_byte_entry); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump[0]); exit_invalid[7] = CMP(SLJIT_GREATER, TMP2, 0, STR_PTR, 0); @@ -4571,7 +4911,7 @@ for (i = 0; i < 8; i++) sljit_set_label(exit_invalid[i], exit_invalid_label); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } #endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ @@ -4601,13 +4941,13 @@ OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x10000); exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x400); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(exit_invalid[0]); JUMPHERE(exit_invalid[1]); JUMPHERE(exit_invalid[2]); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfreadnewline_invalid(compiler_common *common) @@ -4634,12 +4974,12 @@ OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(exit_invalid[0]); JUMPHERE(exit_invalid[1]); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfmoveback_invalid(compiler_common *common) @@ -4659,7 +4999,7 @@ exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x400); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(exit_invalid[0]); JUMPHERE(exit_invalid[1]); @@ -4667,12 +5007,12 @@ JUMPHERE(exit_invalid[2]); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_utfpeakcharback_invalid(compiler_common *common) { -/* Peak a character back. */ +/* Peak a character back. Does not modify STR_PTR. */ DEFINE_COMPILER; struct sljit_jump *jump; struct sljit_jump *exit_invalid[3]; @@ -4692,14 +5032,14 @@ OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); JUMPHERE(jump); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(exit_invalid[0]); JUMPHERE(exit_invalid[1]); JUMPHERE(exit_invalid[2]); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } #endif /* PCRE2_CODE_UNIT_WIDTH == 16 */ @@ -4745,7 +5085,7 @@ OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_getucdtype(compiler_common *common) @@ -4786,19 +5126,13 @@ OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); -// PH hacking -//fprintf(stderr, "~~A\n"); - OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); - +/* TMP2 is multiplied by 12. Same as (TMP2 << 2) + ((TMP2 << 2) << 1). */ OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 2); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 1); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); - -// OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } #endif /* SUPPORT_UNICODE */ @@ -4866,15 +5200,27 @@ else if ((overall_options & PCRE2_USE_OFFSET_LIMIT) != 0) /* Check whether offset limit is set and valid. */ SLJIT_ASSERT(common->match_end_ptr != 0); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, offset_limit)); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, offset_limit)); + } + else + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, offset_limit)); + OP1(SLJIT_MOV, TMP2, 0, STR_END, 0); end = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw) PCRE2_UNSET); - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + if (HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + else + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); + #if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); + if (HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0); OP1(SLJIT_MOV, TMP2, 0, STR_END, 0); @@ -5074,6 +5420,8 @@ while (TRUE) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: cc = bracketend(cc); continue; @@ -5373,7 +5721,12 @@ while (TRUE) #endif { chr = *cc; - othercase[0] = TABLE_GET(chr, common->fcc, chr); +#ifdef SUPPORT_UNICODE + if (common->ucp && chr > 127) + othercase[0] = UCD_OTHERCASE(chr); + else +#endif + othercase[0] = TABLE_GET(chr, common->fcc, chr); } } else @@ -5434,699 +5787,56 @@ CMPTO(SLJIT_EQUAL, reg, 0, SLJIT_IMM, 0xdc00, label); } #endif -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) +#include "pcre2_jit_simd_inc.h" -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -static struct sljit_jump *jump_if_utf_char_start(struct sljit_compiler *compiler, sljit_s32 reg) -{ -#if PCRE2_CODE_UNIT_WIDTH == 8 -OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xc0); -return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0x80); -#elif PCRE2_CODE_UNIT_WIDTH == 16 -OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xfc00); -return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0xdc00); -#else -#error "Unknown code width" -#endif -} -#endif - -static sljit_s32 character_to_int32(PCRE2_UCHAR chr) -{ -sljit_u32 value = chr; -#if PCRE2_CODE_UNIT_WIDTH == 8 -#define SSE2_COMPARE_TYPE_INDEX 0 -return (sljit_s32)((value << 24) | (value << 16) | (value << 8) | value); -#elif PCRE2_CODE_UNIT_WIDTH == 16 -#define SSE2_COMPARE_TYPE_INDEX 1 -return (sljit_s32)((value << 16) | value); -#elif PCRE2_CODE_UNIT_WIDTH == 32 -#define SSE2_COMPARE_TYPE_INDEX 2 -return (sljit_s32)(value); -#else -#error "Unsupported unit width" -#endif -} +#ifdef JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD -static void load_from_mem_sse2(struct sljit_compiler *compiler, sljit_s32 dst_xmm_reg, sljit_s32 src_general_reg) +static BOOL check_fast_forward_char_pair_simd(compiler_common *common, fast_forward_char_data *chars, int max) { -#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) -sljit_u8 instruction[5]; -#else -sljit_u8 instruction[4]; -#endif - -SLJIT_ASSERT(dst_xmm_reg < 8); - -/* MOVDQA xmm1, xmm2/m128 */ -#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) -if (src_general_reg < 8) - { - instruction[0] = 0x66; - instruction[1] = 0x0f; - instruction[2] = 0x6f; - instruction[3] = (dst_xmm_reg << 3) | src_general_reg; - sljit_emit_op_custom(compiler, instruction, 4); - } -else - { - instruction[0] = 0x66; - instruction[1] = 0x41; - instruction[2] = 0x0f; - instruction[3] = 0x6f; - instruction[4] = (dst_xmm_reg << 3) | (src_general_reg & 0x7); - sljit_emit_op_custom(compiler, instruction, 4); - } -#else -instruction[0] = 0x66; -instruction[1] = 0x0f; -instruction[2] = 0x6f; -instruction[3] = (dst_xmm_reg << 3) | src_general_reg; -sljit_emit_op_custom(compiler, instruction, 4); -#endif -} - -static void fast_forward_char_pair_sse2_compare(struct sljit_compiler *compiler, PCRE2_UCHAR char1, PCRE2_UCHAR char2, - sljit_u32 bit, sljit_s32 dst_ind, sljit_s32 cmp1_ind, sljit_s32 cmp2_ind, sljit_s32 tmp_ind) -{ -sljit_u8 instruction[4]; -instruction[0] = 0x66; -instruction[1] = 0x0f; - -if (char1 == char2 || bit != 0) - { - if (bit != 0) - { - /* POR xmm1, xmm2/m128 */ - /* instruction[0] = 0x66; */ - /* instruction[1] = 0x0f; */ - instruction[2] = 0xeb; - instruction[3] = 0xc0 | (dst_ind << 3) | cmp2_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - - /* PCMPEQB/W/D xmm1, xmm2/m128 */ - /* instruction[0] = 0x66; */ - /* instruction[1] = 0x0f; */ - instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; - instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } -else - { - /* MOVDQA xmm1, xmm2/m128 */ - /* instruction[0] = 0x66; */ - /* instruction[1] = 0x0f; */ - instruction[2] = 0x6f; - instruction[3] = 0xc0 | (tmp_ind << 3) | dst_ind; - sljit_emit_op_custom(compiler, instruction, 4); - - /* PCMPEQB/W/D xmm1, xmm2/m128 */ - /* instruction[0] = 0x66; */ - /* instruction[1] = 0x0f; */ - instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; - instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind; - sljit_emit_op_custom(compiler, instruction, 4); - - instruction[3] = 0xc0 | (tmp_ind << 3) | cmp2_ind; - sljit_emit_op_custom(compiler, instruction, 4); - - /* POR xmm1, xmm2/m128 */ - /* instruction[0] = 0x66; */ - /* instruction[1] = 0x0f; */ - instruction[2] = 0xeb; - instruction[3] = 0xc0 | (dst_ind << 3) | tmp_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } -} - -static void fast_forward_first_char2_sse2(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset) -{ -DEFINE_COMPILER; -struct sljit_label *start; -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -struct sljit_label *restart; -#endif -struct sljit_jump *quit; -struct sljit_jump *partial_quit[2]; -sljit_u8 instruction[8]; -sljit_s32 tmp1_ind = sljit_get_register_index(TMP1); -sljit_s32 str_ptr_ind = sljit_get_register_index(STR_PTR); -sljit_s32 data_ind = 0; -sljit_s32 tmp_ind = 1; -sljit_s32 cmp1_ind = 2; -sljit_s32 cmp2_ind = 3; -sljit_u32 bit = 0; - -SLJIT_UNUSED_ARG(offset); - -if (char1 != char2) - { - bit = char1 ^ char2; - if (!is_powerof2(bit)) - bit = 0; - } - -partial_quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); -if (common->mode == PCRE2_JIT_COMPLETE) - add_jump(compiler, &common->failed_match, partial_quit[0]); - -/* First part (unaligned start) */ - -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit)); - -SLJIT_ASSERT(tmp1_ind < 8); - -/* MOVD xmm, r/m32 */ -instruction[0] = 0x66; -instruction[1] = 0x0f; -instruction[2] = 0x6e; -instruction[3] = 0xc0 | (cmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (char1 != char2) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2)); - - /* MOVD xmm, r/m32 */ - instruction[3] = 0xc0 | (cmp2_ind << 3) | tmp1_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); - -/* PSHUFD xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x70; -instruction[3] = 0xc0 | (cmp1_ind << 3) | 2; -instruction[4] = 0; -sljit_emit_op_custom(compiler, instruction, 5); - -if (char1 != char2) - { - /* PSHUFD xmm1, xmm2/m128, imm8 */ - instruction[3] = 0xc0 | (cmp2_ind << 3) | 3; - sljit_emit_op_custom(compiler, instruction, 5); - } - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -restart = LABEL(); -#endif -OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf); - -load_from_mem_sse2(compiler, data_ind, str_ptr_ind); -fast_forward_char_pair_sse2_compare(compiler, char1, char2, bit, data_ind, cmp1_ind, cmp2_ind, tmp_ind); - -/* PMOVMSKB reg, xmm */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); -sljit_set_current_flags(compiler, SLJIT_SET_Z); - -quit = JUMP(SLJIT_NOT_ZERO); - -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - -start = LABEL(); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); - -partial_quit[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); -if (common->mode == PCRE2_JIT_COMPLETE) - add_jump(compiler, &common->failed_match, partial_quit[1]); - -/* Second part (aligned) */ - -load_from_mem_sse2(compiler, 0, str_ptr_ind); -fast_forward_char_pair_sse2_compare(compiler, char1, char2, bit, data_ind, cmp1_ind, cmp2_ind, tmp_ind); - -/* PMOVMSKB reg, xmm */ -instruction[0] = 0x66; -instruction[1] = 0x0f; -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); -sljit_set_current_flags(compiler, SLJIT_SET_Z); - -JUMPTO(SLJIT_ZERO, start); - -JUMPHERE(quit); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - -if (common->mode != PCRE2_JIT_COMPLETE) - { - JUMPHERE(partial_quit[0]); - JUMPHERE(partial_quit[1]); - OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); - CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0); - } -else - add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -if (common->utf && offset > 0) - { - SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE); - - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); - - quit = jump_if_utf_char_start(compiler, TMP1); - - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, restart); - - JUMPHERE(quit); - } -#endif -} - -#ifndef _WIN64 - -static SLJIT_INLINE sljit_u32 max_fast_forward_char_pair_sse2_offset(void) -{ -#if PCRE2_CODE_UNIT_WIDTH == 8 -return 15; -#elif PCRE2_CODE_UNIT_WIDTH == 16 -return 7; -#elif PCRE2_CODE_UNIT_WIDTH == 32 -return 3; -#else -#error "Unsupported unit width" -#endif -} - -static void fast_forward_char_pair_sse2(compiler_common *common, sljit_s32 offs1, - PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b) -{ -DEFINE_COMPILER; -sljit_u32 bit1 = 0; -sljit_u32 bit2 = 0; -sljit_u32 diff = IN_UCHARS(offs1 - offs2); -sljit_s32 tmp1_ind = sljit_get_register_index(TMP1); -sljit_s32 tmp2_ind = sljit_get_register_index(TMP2); -sljit_s32 str_ptr_ind = sljit_get_register_index(STR_PTR); -sljit_s32 data1_ind = 0; -sljit_s32 data2_ind = 1; -sljit_s32 tmp_ind = 2; -sljit_s32 cmp1a_ind = 3; -sljit_s32 cmp1b_ind = 4; -sljit_s32 cmp2a_ind = 5; -sljit_s32 cmp2b_ind = 6; -struct sljit_label *start; -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -struct sljit_label *restart; -#endif -struct sljit_jump *jump[2]; - -sljit_u8 instruction[8]; - -SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2); -SLJIT_ASSERT(diff <= IN_UCHARS(max_fast_forward_char_pair_sse2_offset())); -SLJIT_ASSERT(tmp1_ind < 8 && tmp2_ind == 1); - -/* Initialize. */ -if (common->match_end_ptr != 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1)); - - OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, STR_END, 0); - CMOV(SLJIT_LESS, STR_END, TMP1, 0); - } - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1)); -add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - -/* MOVD xmm, r/m32 */ -instruction[0] = 0x66; -instruction[1] = 0x0f; -instruction[2] = 0x6e; - -if (char1a == char1b) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a)); -else - { - bit1 = char1a ^ char1b; - if (is_powerof2(bit1)) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a | bit1)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit1)); - } - else - { - bit1 = 0; - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char1b)); - } - } - -instruction[3] = 0xc0 | (cmp1a_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (char1a != char1b) - { - instruction[3] = 0xc0 | (cmp1b_ind << 3) | tmp2_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -if (char2a == char2b) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a)); -else - { - bit2 = char2a ^ char2b; - if (is_powerof2(bit2)) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a | bit2)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit2)); - } - else - { - bit2 = 0; - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char2b)); - } - } - -instruction[3] = 0xc0 | (cmp2a_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (char2a != char2b) - { - instruction[3] = 0xc0 | (cmp2b_ind << 3) | tmp2_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PSHUFD xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x70; -instruction[4] = 0; - -instruction[3] = 0xc0 | (cmp1a_ind << 3) | cmp1a_ind; -sljit_emit_op_custom(compiler, instruction, 5); - -if (char1a != char1b) - { - instruction[3] = 0xc0 | (cmp1b_ind << 3) | cmp1b_ind; - sljit_emit_op_custom(compiler, instruction, 5); - } - -instruction[3] = 0xc0 | (cmp2a_ind << 3) | cmp2a_ind; -sljit_emit_op_custom(compiler, instruction, 5); - -if (char2a != char2b) - { - instruction[3] = 0xc0 | (cmp2b_ind << 3) | cmp2b_ind; - sljit_emit_op_custom(compiler, instruction, 5); - } - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -restart = LABEL(); -#endif - -OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1 - offs2)); -OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); -OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, ~0xf); - -load_from_mem_sse2(compiler, data1_ind, str_ptr_ind); - -jump[0] = CMP(SLJIT_EQUAL, STR_PTR, 0, TMP1, 0); - -load_from_mem_sse2(compiler, data2_ind, tmp1_ind); - -/* MOVDQA xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x6f; -instruction[3] = 0xc0 | (tmp_ind << 3) | data1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -/* PSLLDQ xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x73; -instruction[3] = 0xc0 | (7 << 3) | tmp_ind; -instruction[4] = diff; -sljit_emit_op_custom(compiler, instruction, 5); - -/* PSRLDQ xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -/* instruction[2] = 0x73; */ -instruction[3] = 0xc0 | (3 << 3) | data2_ind; -instruction[4] = 16 - diff; -sljit_emit_op_custom(compiler, instruction, 5); - -/* POR xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xeb; -instruction[3] = 0xc0 | (data2_ind << 3) | tmp_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -jump[1] = JUMP(SLJIT_JUMP); - -JUMPHERE(jump[0]); - -/* MOVDQA xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x6f; -instruction[3] = 0xc0 | (data2_ind << 3) | data1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -/* PSLLDQ xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x73; -instruction[3] = 0xc0 | (7 << 3) | data2_ind; -instruction[4] = diff; -sljit_emit_op_custom(compiler, instruction, 5); - -JUMPHERE(jump[1]); - -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf); - -fast_forward_char_pair_sse2_compare(compiler, char2a, char2b, bit2, data2_ind, cmp2a_ind, cmp2b_ind, tmp_ind); -fast_forward_char_pair_sse2_compare(compiler, char1a, char1b, bit1, data1_ind, cmp1a_ind, cmp1b_ind, tmp_ind); - -/* PAND xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xdb; -instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -/* PMOVMSKB reg, xmm */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -/* Ignore matches before the first STR_PTR. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); -sljit_set_current_flags(compiler, SLJIT_SET_Z); - -jump[0] = JUMP(SLJIT_NOT_ZERO); - -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - -/* Main loop. */ -instruction[0] = 0x66; -instruction[1] = 0x0f; - -start = LABEL(); - -load_from_mem_sse2(compiler, data2_ind, str_ptr_ind); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); -add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - -load_from_mem_sse2(compiler, data1_ind, str_ptr_ind); - -/* PSRLDQ xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x73; -instruction[3] = 0xc0 | (3 << 3) | data2_ind; -instruction[4] = 16 - diff; -sljit_emit_op_custom(compiler, instruction, 5); - -/* MOVDQA xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x6f; -instruction[3] = 0xc0 | (tmp_ind << 3) | data1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -/* PSLLDQ xmm1, xmm2/m128, imm8 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0x73; -instruction[3] = 0xc0 | (7 << 3) | tmp_ind; -instruction[4] = diff; -sljit_emit_op_custom(compiler, instruction, 5); - -/* POR xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xeb; -instruction[3] = 0xc0 | (data2_ind << 3) | tmp_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -fast_forward_char_pair_sse2_compare(compiler, char1a, char1b, bit1, data1_ind, cmp1a_ind, cmp1b_ind, tmp_ind); -fast_forward_char_pair_sse2_compare(compiler, char2a, char2b, bit2, data2_ind, cmp2a_ind, cmp2b_ind, tmp_ind); - -/* PAND xmm1, xmm2/m128 */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xdb; -instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -/* PMOVMSKB reg, xmm */ -/* instruction[0] = 0x66; */ -/* instruction[1] = 0x0f; */ -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); -sljit_set_current_flags(compiler, SLJIT_SET_Z); - -JUMPTO(SLJIT_ZERO, start); - -JUMPHERE(jump[0]); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - -add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - -if (common->match_end_ptr != 0) - OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -if (common->utf) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offs1)); - - jump[0] = jump_if_utf_char_start(compiler, TMP1); - - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, restart); - - add_jump(compiler, &common->failed_match, JUMP(SLJIT_JUMP)); - - JUMPHERE(jump[0]); - } -#endif - -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1)); - -if (common->match_end_ptr != 0) - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); -} - -static BOOL check_fast_forward_char_pair_sse2(compiler_common *common, fast_forward_char_data *chars, int max) -{ -sljit_s32 i, j, priority, count; -sljit_u32 priorities; -PCRE2_UCHAR a1, a2, b1, b2; - -priorities = 0; - -count = 0; -for (i = 0; i < max; i++) - { - if (chars[i].last_count > 2) - { - SLJIT_ASSERT(chars[i].last_count <= 7); - - priorities |= (1 << chars[i].last_count); - count++; - } - } - -if (count < 2) - return FALSE; - -for (priority = 7; priority > 2; priority--) - { - if ((priorities & (1 << priority)) == 0) - continue; + sljit_s32 i, j, max_i = 0, max_j = 0; + sljit_u32 max_pri = 0; + PCRE2_UCHAR a1, a2, a_pri, b1, b2, b_pri; for (i = max - 1; i >= 1; i--) - if (chars[i].last_count >= priority) + { + if (chars[i].last_count > 2) { - SLJIT_ASSERT(chars[i].count <= 2 && chars[i].count >= 1); - a1 = chars[i].chars[0]; a2 = chars[i].chars[1]; + a_pri = chars[i].last_count; - j = i - max_fast_forward_char_pair_sse2_offset(); + j = i - max_fast_forward_char_pair_offset(); if (j < 0) j = 0; while (j < i) { - if (chars[j].last_count >= priority) + b_pri = chars[j].last_count; + if (b_pri > 2 && a_pri + b_pri >= max_pri) { b1 = chars[j].chars[0]; b2 = chars[j].chars[1]; if (a1 != b1 && a1 != b2 && a2 != b1 && a2 != b2) { - fast_forward_char_pair_sse2(common, i, a1, a2, j, b1, b2); - return TRUE; + max_pri = a_pri + b_pri; + max_i = i; + max_j = j; } } j++; } } - } - -return FALSE; -} + } -#endif +if (max_pri == 0) + return FALSE; -#undef SSE2_COMPARE_TYPE_INDEX +fast_forward_char_pair_simd(common, max_i, chars[max_i].chars[0], chars[max_i].chars[1], max_j, chars[max_j].chars[0], chars[max_j].chars[1]); +return TRUE; +} -#endif +#endif /* JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD */ static void fast_forward_first_char2(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset) { @@ -6154,13 +5864,11 @@ if (has_match_end) CMOV(SLJIT_GREATER, STR_END, TMP1, 0); } -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) +#ifdef JIT_HAS_FAST_FORWARD_CHAR_SIMD -/* SSE2 accelerated first character search. */ - -if (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) +if (JIT_HAS_FAST_FORWARD_CHAR_SIMD) { - fast_forward_first_char2_sse2(common, char1, char2, offset); + fast_forward_char_simd(common, char1, char2, offset); if (offset > 0) OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); @@ -6267,8 +5975,8 @@ for (i = 0; i < max; i++) chars[i].last_count = (chars[i].count == 255) ? 0 : 1; } -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) && !(defined _WIN64) -if (sljit_has_cpu_feature(SLJIT_HAS_SSE2) && check_fast_forward_char_pair_sse2(common, chars, max)) +#ifdef JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD +if (JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD && check_fast_forward_char_pair_simd(common, chars, max)) return TRUE; #endif @@ -6353,18 +6061,21 @@ if (common->match_end_ptr != 0) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); - OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + OP2(SLJIT_SUB | SLJIT_SET_LESS, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + add_jump(compiler, &common->failed_match, JUMP(SLJIT_LESS)); OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_END, 0, TMP1, 0); CMOV(SLJIT_GREATER, STR_END, TMP1, 0); } else - OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + add_jump(compiler, &common->failed_match, JUMP(SLJIT_LESS)); + } SLJIT_ASSERT(range_right >= 0); -#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -OP1(SLJIT_MOV, RETURN_ADDR, 0, SLJIT_IMM, (sljit_sw)update_table); -#endif +if (!HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, RETURN_ADDR, 0, SLJIT_IMM, (sljit_sw)update_table); start = LABEL(); add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); @@ -6375,11 +6086,11 @@ OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right)); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right + 1) - 1); #endif -#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(RETURN_ADDR, TMP1), 0); -#else -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)update_table); -#endif +if (!HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(RETURN_ADDR, TMP1), 0); +else + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)update_table); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, start); @@ -6444,8 +6155,8 @@ oc = first_char; if ((common->re->flags & PCRE2_FIRSTCASELESS) != 0) { oc = TABLE_GET(first_char, common->fcc, first_char); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (first_char > 127 && common->utf) +#if defined SUPPORT_UNICODE + if (first_char > 127 && (common->utf || common->ucp)) oc = UCD_OTHERCASE(first_char); #endif } @@ -6457,9 +6168,9 @@ static SLJIT_INLINE void fast_forward_newline(compiler_common *common) { DEFINE_COMPILER; struct sljit_label *loop; -struct sljit_jump *lastchar; +struct sljit_jump *lastchar = NULL; struct sljit_jump *firstchar; -struct sljit_jump *quit; +struct sljit_jump *quit = NULL; struct sljit_jump *foundcr = NULL; struct sljit_jump *notfoundnl; jump_list *newline = NULL; @@ -6472,57 +6183,140 @@ if (common->match_end_ptr != 0) if (common->nltype == NLTYPE_FIXED && common->newline > 255) { - lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); +#ifdef JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD + if (JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD && common->mode == PCRE2_JIT_COMPLETE) + { + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); + } + firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_NOT_EQUAL); #if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); #endif - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - loop = LABEL(); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop); - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop); + fast_forward_char_pair_simd(common, 1, common->newline & 0xff, common->newline & 0xff, 0, (common->newline >> 8) & 0xff, (common->newline >> 8) & 0xff); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + } + else +#endif /* JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD */ + { + lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); + } + firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); + + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL); +#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + + loop = LABEL(); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop); + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop); + + JUMPHERE(quit); + JUMPHERE(lastchar); + } - JUMPHERE(quit); JUMPHERE(firstchar); - JUMPHERE(lastchar); if (common->match_end_ptr != 0) OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); return; } -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + } +else + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str)); + /* Example: match /^/ to \r\n from offset 1. */ -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); -move_back(common, NULL, FALSE); + +if (common->nltype == NLTYPE_ANY) + move_back(common, NULL, FALSE); +else + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); loop = LABEL(); common->ff_newline_shortcut = loop; -read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE); -lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); -if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) - foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); -check_newlinechar(common, common->nltype, &newline, FALSE); -set_jumps(newline, loop); +#ifdef JIT_HAS_FAST_FORWARD_CHAR_SIMD +if (JIT_HAS_FAST_FORWARD_CHAR_SIMD && (common->nltype == NLTYPE_FIXED || common->nltype == NLTYPE_ANYCRLF)) + { + if (common->nltype == NLTYPE_ANYCRLF) + { + fast_forward_char_simd(common, CHAR_CR, CHAR_LF, 0); + if (common->mode != PCRE2_JIT_COMPLETE) + lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + quit = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); + } + else + { + fast_forward_char_simd(common, common->newline, common->newline, 0); + + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + if (common->mode != PCRE2_JIT_COMPLETE) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0); + } + } + } +else +#endif /* JIT_HAS_FAST_FORWARD_CHAR_SIMD */ + { + read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE); + lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) + foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); + check_newlinechar(common, common->nltype, &newline, FALSE); + set_jumps(newline, loop); + } if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) { - quit = JUMP(SLJIT_JUMP); - JUMPHERE(foundcr); + if (quit == NULL) + { + quit = JUMP(SLJIT_JUMP); + JUMPHERE(foundcr); + } + notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); @@ -6534,7 +6328,9 @@ if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) JUMPHERE(notfoundnl); JUMPHERE(quit); } -JUMPHERE(lastchar); + +if (lastchar) + JUMPHERE(lastchar); JUMPHERE(firstchar); if (common->match_end_ptr != 0) @@ -6586,7 +6382,7 @@ if (!optimize_class(common, start_bits, (start_bits[31] & 0x80) != 0, FALSE, &ma OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits); - if (sljit_get_register_index(TMP3) >= 0) + if (!HAS_VIRTUAL_REGISTERS) { OP2(SLJIT_SHL, TMP3, 0, SLJIT_IMM, 1, TMP2, 0); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP3, 0); @@ -6615,67 +6411,80 @@ if (common->match_end_ptr != 0) OP1(SLJIT_MOV, STR_END, 0, RETURN_ADDR, 0); } -static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, PCRE2_UCHAR req_char, BOOL caseless, BOOL has_firstchar) +static SLJIT_INLINE jump_list *search_requested_char(compiler_common *common, PCRE2_UCHAR req_char, BOOL caseless, BOOL has_firstchar) { DEFINE_COMPILER; struct sljit_label *loop; struct sljit_jump *toolong; -struct sljit_jump *alreadyfound; +struct sljit_jump *already_found; struct sljit_jump *found; -struct sljit_jump *foundoc = NULL; -struct sljit_jump *notfound; +struct sljit_jump *found_oc = NULL; +jump_list *not_found = NULL; sljit_u32 oc, bit; SLJIT_ASSERT(common->req_char_ptr != 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr); -OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_CU_MAX); -toolong = CMP(SLJIT_LESS, TMP1, 0, STR_END, 0); -alreadyfound = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); +OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(REQ_CU_MAX) * 100); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr); +toolong = CMP(SLJIT_LESS, TMP2, 0, STR_END, 0); +already_found = CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0); if (has_firstchar) OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); else OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0); -loop = LABEL(); -notfound = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0); - -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0); oc = req_char; if (caseless) { oc = TABLE_GET(req_char, common->fcc, req_char); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (req_char > 127 && common->utf) +#if defined SUPPORT_UNICODE + if (req_char > 127 && (common->utf || common->ucp)) oc = UCD_OTHERCASE(req_char); #endif } -if (req_char == oc) - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); + +#ifdef JIT_HAS_FAST_REQUESTED_CHAR_SIMD +if (JIT_HAS_FAST_REQUESTED_CHAR_SIMD) + { + not_found = fast_requested_char_simd(common, req_char, oc); + } else +#endif { - bit = req_char ^ oc; - if (is_powerof2(bit)) - { - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit); - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit); - } + loop = LABEL(); + add_jump(compiler, ¬_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0)); + + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0); + + if (req_char == oc) + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); else { - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); - foundoc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc); + bit = req_char ^ oc; + if (is_powerof2(bit)) + { + OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit); + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit); + } + else + { + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); + found_oc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc); + } } + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPTO(SLJIT_JUMP, loop); + + JUMPHERE(found); + if (found_oc) + JUMPHERE(found_oc); } -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); -JUMPTO(SLJIT_JUMP, loop); -JUMPHERE(found); -if (foundoc) - JUMPHERE(foundoc); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, TMP1, 0); -JUMPHERE(alreadyfound); + +JUMPHERE(already_found); JUMPHERE(toolong); -return notfound; +return not_found; } static void do_revertframes(compiler_common *common) @@ -6693,7 +6502,7 @@ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw)); jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); -if (sljit_get_register_index(TMP3) < 0) +if (HAS_VIRTUAL_REGISTERS) { OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -(2 * sizeof(sljit_sw))); OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -(3 * sizeof(sljit_sw))); @@ -6713,12 +6522,12 @@ JUMPTO(SLJIT_JUMP, mainloop); JUMPHERE(jump); jump = CMP(SLJIT_NOT_ZERO /* SIG_LESS */, TMP2, 0, SLJIT_IMM, 0); /* End of reverting values. */ -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); JUMPHERE(jump); OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); -if (sljit_get_register_index(TMP3) < 0) +if (HAS_VIRTUAL_REGISTERS) { OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -(2 * sizeof(sljit_sw))); OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); @@ -6737,7 +6546,11 @@ static void check_wordboundary(compiler_common *common) DEFINE_COMPILER; struct sljit_jump *skipread; jump_list *skipread_list = NULL; -jump_list *invalid_utf = NULL; +#ifdef SUPPORT_UNICODE +struct sljit_label *valid_utf; +jump_list *invalid_utf1 = NULL; +#endif /* SUPPORT_UNICODE */ +jump_list *invalid_utf2 = NULL; #if PCRE2_CODE_UNIT_WIDTH != 8 || defined SUPPORT_UNICODE struct sljit_jump *jump; #endif /* PCRE2_CODE_UNIT_WIDTH != 8 || SUPPORT_UNICODE */ @@ -6751,19 +6564,37 @@ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); -if (common->mode == PCRE2_JIT_COMPLETE) - peek_char_back(common, READ_CHAR_MAX, &invalid_utf); +#ifdef SUPPORT_UNICODE +if (common->invalid_utf) + { + peek_char_back(common, READ_CHAR_MAX, &invalid_utf1); + + if (common->mode != PCRE2_JIT_COMPLETE) + { + OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); + move_back(common, NULL, TRUE); + check_start_used_ptr(common); + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + OP1(SLJIT_MOV, STR_PTR, 0, TMP2, 0); + } + } else +#endif /* SUPPORT_UNICODE */ { - move_back(common, &invalid_utf, FALSE); - check_start_used_ptr(common); - /* No need precise read since match fails anyway. */ - read_char(common, 0, READ_CHAR_MAX, &invalid_utf, READ_CHAR_UPDATE_STR_PTR); + if (common->mode == PCRE2_JIT_COMPLETE) + peek_char_back(common, READ_CHAR_MAX, NULL); + else + { + move_back(common, NULL, TRUE); + check_start_used_ptr(common); + read_char(common, 0, READ_CHAR_MAX, NULL, READ_CHAR_UPDATE_STR_PTR); + } } /* Testing char type. */ #ifdef SUPPORT_UNICODE -if (common->use_ucp) +if (common->ucp) { OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); @@ -6802,11 +6633,14 @@ JUMPHERE(skipread); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); check_str_end(common, &skipread_list); -peek_char(common, READ_CHAR_MAX, SLJIT_MEM1(SLJIT_SP), LOCALS1, &invalid_utf); +peek_char(common, READ_CHAR_MAX, SLJIT_MEM1(SLJIT_SP), LOCALS1, &invalid_utf2); /* Testing char type. This is a code duplication. */ #ifdef SUPPORT_UNICODE -if (common->use_ucp) + +valid_utf = LABEL(); + +if (common->ucp) { OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); @@ -6846,18 +6680,24 @@ set_jumps(skipread_list, LABEL()); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); OP2(SLJIT_XOR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, TMP3, 0); -sljit_emit_fast_return(compiler, TMP1, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); #ifdef SUPPORT_UNICODE if (common->invalid_utf) { - SLJIT_ASSERT(invalid_utf != NULL); + set_jumps(invalid_utf1, LABEL()); + + peek_char(common, READ_CHAR_MAX, SLJIT_MEM1(SLJIT_SP), LOCALS1, NULL); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR, valid_utf); - set_jumps(invalid_utf, LABEL()); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, -1); - sljit_emit_fast_return(compiler, TMP1, 0); - return; + OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); + + set_jumps(invalid_utf2, LABEL()); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + OP1(SLJIT_MOV, TMP2, 0, TMP3, 0); + OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); } #endif /* SUPPORT_UNICODE */ } @@ -7147,7 +6987,7 @@ if (common->utf) #endif #endif /* SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH == [16|32] */ OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void check_hspace(compiler_common *common) @@ -7186,7 +7026,7 @@ if (common->utf) #endif /* SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH == [16|32] */ OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void check_vspace(compiler_common *common) @@ -7214,7 +7054,7 @@ if (common->utf) #endif /* SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH == [16|32] */ OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0); } static void do_casefulcmp(compiler_common *common) @@ -7225,7 +7065,7 @@ struct sljit_label *label; int char1_reg; int char2_reg; -if (sljit_get_register_index(TMP3) < 0) +if (HAS_VIRTUAL_REGISTERS) { char1_reg = STR_END; char2_reg = STACK_TOP; @@ -7294,7 +7134,7 @@ if (char1_reg == STR_END) OP1(SLJIT_MOV, char2_reg, 0, RETURN_ADDR, 0); } -sljit_emit_fast_return(compiler, TMP1, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); } static void do_caselesscmp(compiler_common *common) @@ -7307,7 +7147,7 @@ int char2_reg; int lcc_table; int opt_type = 0; -if (sljit_get_register_index(TMP3) < 0) +if (HAS_VIRTUAL_REGISTERS) { char2_reg = STACK_TOP; lcc_table = STACK_LIMIT; @@ -7392,7 +7232,7 @@ if (char2_reg == STACK_TOP) } OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, TMP1, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); } static PCRE2_SPTR byte_sequence_compare(compiler_common *common, BOOL caseless, PCRE2_SPTR cc, @@ -7703,7 +7543,13 @@ cc = ccbegin; if ((cc[-1] & XCL_NOT) != 0) read_char(common, min, max, backtracks, READ_CHAR_UPDATE_STR_PTR); else + { +#ifdef SUPPORT_UNICODE + read_char(common, min, max, (needstype || needsscript) ? backtracks : NULL, 0); +#else /* !SUPPORT_UNICODE */ read_char(common, min, max, NULL, 0); +#endif /* SUPPORT_UNICODE */ + } if ((cc[-1] & XCL_HASPROP) == 0) { @@ -7789,18 +7635,11 @@ if (needstype || needsscript) /* Before anything else, we deal with scripts. */ if (needsscript) { -// PH hacking -//fprintf(stderr, "~~B\n"); - - OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); - - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script)); - - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 3); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); - // OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script)); ccbegin = cc; @@ -7835,43 +7674,52 @@ if (needstype || needsscript) } cc = ccbegin; - } - if (needschar) - OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + if (needstype) + { + /* TMP2 has already been shifted by 2 */ + if (!needschar) + { + OP2(SLJIT_ADD, TMP1, 0, TMP2, 0, TMP2, 0); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); + } + else + { + OP2(SLJIT_ADD, TMP1, 0, TMP2, 0, TMP2, 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); - if (needstype) + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); + typereg = RETURN_ADDR; + } + } + else if (needschar) + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + } + else if (needstype) { + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 3); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 2); + if (!needschar) { -// PH hacking -//fprintf(stderr, "~~C\n"); - OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); - OP2(SLJIT_ADD, TMP1, 0, TMP2, 0, TMP1, 0); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); - - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); - -// OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); } else { -// PH hacking -//fprintf(stderr, "~~D\n"); - OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); - - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); typereg = RETURN_ADDR; } } + else if (needschar) + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); } #endif /* SUPPORT_UNICODE */ @@ -8174,14 +8022,24 @@ struct sljit_label *label; switch(type) { case OP_SOD: - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + } + else + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); return cc; case OP_SOM: - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + } + else + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str)); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); return cc; @@ -8191,9 +8049,7 @@ switch(type) #ifdef SUPPORT_UNICODE if (common->invalid_utf) { - OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0); - add_jump(compiler, backtracks, JUMP(SLJIT_SIG_LESS)); - add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + add_jump(compiler, backtracks, CMP((type == OP_NOT_WORD_BOUNDARY) ? SLJIT_NOT_EQUAL : SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0)); return cc; } #endif /* SUPPORT_UNICODE */ @@ -8267,17 +8123,24 @@ switch(type) JUMPHERE(jump[3]); } JUMPHERE(jump[0]); - check_partial(common, FALSE); + if (common->mode != PCRE2_JIT_COMPLETE) + check_partial(common, TRUE); return cc; case OP_EOD: add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0)); - check_partial(common, FALSE); + if (common->mode != PCRE2_JIT_COMPLETE) + check_partial(common, TRUE); return cc; case OP_DOLL: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); + } + else + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); if (!common->endonly) @@ -8291,8 +8154,13 @@ switch(type) case OP_DOLLM: jump[1] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); + } + else + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTEOL); add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); check_partial(common, FALSE); jump[0] = JUMP(SLJIT_JUMP); @@ -8327,18 +8195,38 @@ switch(type) return cc; case OP_CIRC: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0)); - OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); - add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0)); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); + } + else + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0)); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); + } return cc; case OP_CIRCM: - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP2, 0); - OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + /* TMP2 might be used by peek_char_back. */ + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP2, 0); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); + jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP2, 0); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + } add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); jump[0] = JUMP(SLJIT_JUMP); JUMPHERE(jump[1]); @@ -8367,11 +8255,16 @@ switch(type) length = GET(cc, 0); if (length == 0) return cc + LINK_SIZE; - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + } + else + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin)); #ifdef SUPPORT_UNICODE if (common->utf) { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, length); label = LABEL(); add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0)); @@ -8382,9 +8275,8 @@ switch(type) else #endif { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0)); } check_start_used_ptr(common); return cc + LINK_SIZE; @@ -8402,12 +8294,12 @@ static PCRE2_SPTR SLJIT_FUNC do_extuni_utf(jit_arguments *args, PCRE2_SPTR cc) PCRE2_SPTR start_subject = args->begin; PCRE2_SPTR end_subject = args->end; int lgb, rgb, ricount; -PCRE2_SPTR prevcc, startcc, bptr; +PCRE2_SPTR prevcc, endcc, bptr; BOOL first = TRUE; uint32_t c; prevcc = cc; -startcc = NULL; +endcc = NULL; do { GETCHARINC(c, cc); @@ -8416,7 +8308,7 @@ do if (first) { lgb = rgb; - startcc = cc; + endcc = cc; first = FALSE; continue; } @@ -8455,25 +8347,27 @@ do lgb != ucp_gbExtended_Pictographic) lgb = rgb; - prevcc = startcc; - startcc = cc; + prevcc = endcc; + endcc = cc; } while (cc < end_subject); -return startcc; +return endcc; } +#endif /* PCRE2_CODE_UNIT_WIDTH != 32 */ + static PCRE2_SPTR SLJIT_FUNC do_extuni_utf_invalid(jit_arguments *args, PCRE2_SPTR cc) { PCRE2_SPTR start_subject = args->begin; PCRE2_SPTR end_subject = args->end; int lgb, rgb, ricount; -PCRE2_SPTR prevcc, startcc, bptr; +PCRE2_SPTR prevcc, endcc, bptr; BOOL first = TRUE; uint32_t c; prevcc = cc; -startcc = NULL; +endcc = NULL; do { GETCHARINC_INVALID(c, cc, end_subject, break); @@ -8482,7 +8376,7 @@ do if (first) { lgb = rgb; - startcc = cc; + endcc = cc; first = FALSE; continue; } @@ -8520,16 +8414,14 @@ do lgb != ucp_gbExtended_Pictographic) lgb = rgb; - prevcc = startcc; - startcc = cc; + prevcc = endcc; + endcc = cc; } while (cc < end_subject); -return startcc; +return endcc; } -#endif /* PCRE2_CODE_UNIT_WIDTH != 32 */ - static PCRE2_SPTR SLJIT_FUNC do_extuni_no_utf(jit_arguments *args, PCRE2_SPTR cc) { PCRE2_SPTR start_subject = args->begin; @@ -8538,7 +8430,10 @@ int lgb, rgb, ricount; PCRE2_SPTR bptr; uint32_t c; -GETCHARINC(c, cc); +/* Patch by PH */ +/* GETCHARINC(c, cc); */ +c = *cc++; + #if PCRE2_CODE_UNIT_WIDTH == 32 if (c >= 0x110000) return NULL; @@ -8800,8 +8695,10 @@ switch(type) if (common->invalid_utf) add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); #else - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_extuni_no_utf)); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, + common->invalid_utf ? SLJIT_FUNC_OFFSET(do_extuni_utf_invalid) : SLJIT_FUNC_OFFSET(do_extuni_no_utf)); + if (!common->utf || common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); #endif OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); @@ -9197,18 +9094,13 @@ if (common->utf && *cc == OP_REFI) CMPTO(SLJIT_EQUAL, TMP1, 0, char1_reg, 0, loop); -// PH hacking -//fprintf(stderr, "~~E\n"); - OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); - + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records)); @@ -10068,7 +9960,8 @@ if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) } else { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0); + SLJIT_ASSERT(extrasize == 3); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), bra == OP_BRAZERO ? STR_PTR : SLJIT_IMM, 0); } } @@ -10400,7 +10293,7 @@ if (opcode == OP_CBRA || opcode == OP_SCBRA) BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; matchingpath += IMM2_SIZE; } -else if (opcode == OP_ONCE || opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_ASSERT_NA || opcode == OP_ASSERTBACK_NA || opcode == OP_ONCE || opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { /* Other brackets simply allocate the next entry. */ private_data_ptr = PRIVATE_DATA(ccbegin); @@ -10585,7 +10478,7 @@ else if (opcode == OP_CBRA || opcode == OP_SCBRA) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); } } -else if (opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_ASSERT_NA || opcode == OP_ASSERTBACK_NA || opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { /* Saving the previous value. */ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); @@ -10711,6 +10604,9 @@ compile_matchingpath(common, matchingpath, cc, backtrack); if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) return NULL; +if (opcode == OP_ASSERT_NA || opcode == OP_ASSERTBACK_NA) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + if (opcode == OP_ONCE) match_once_common(common, ket, BACKTRACK_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); @@ -10759,10 +10655,23 @@ if (ket != OP_KET || bra != OP_BRA) if (offset != 0) stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); +/* Skip and count the other alternatives. */ +i = 1; +while (*cc == OP_ALT) + { + cc += GET(cc, 1); + i++; + } + if (has_alternatives) { if (opcode != OP_ONCE) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + { + if (i <= 3) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + else + BACKTRACK_AS(bracket_backtrack)->u.matching_put_label = sljit_emit_put_label(compiler, SLJIT_MEM1(STACK_TOP), STACK(stacksize)); + } if (ket != OP_KETRMAX) BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); } @@ -10851,9 +10760,6 @@ if (bra == OP_BRAMINZERO) if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO) count_match(common); -/* Skip the other alternatives. */ -while (*cc == OP_ALT) - cc += GET(cc, 1); cc += 1 + LINK_SIZE; if (opcode == OP_ONCE) @@ -11301,8 +11207,8 @@ backtrack_common *backtrack; PCRE2_UCHAR opcode; PCRE2_UCHAR type; sljit_u32 max = 0, exact; -BOOL fast_fail; -sljit_s32 fast_str_ptr; +sljit_s32 early_fail_ptr = PRIVATE_DATA(cc + 1); +sljit_s32 early_fail_type; BOOL charpos_enabled; PCRE2_UCHAR charpos_char; unsigned int charpos_othercasebit; @@ -11316,21 +11222,27 @@ int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr; int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw); int tmp_base, tmp_offset; +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +BOOL use_tmp; +#endif PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL); -fast_str_ptr = PRIVATE_DATA(cc + 1); -fast_fail = TRUE; +early_fail_type = (early_fail_ptr & 0x7); +early_fail_ptr >>= 3; -SLJIT_ASSERT(common->fast_forward_bc_ptr == NULL || fast_str_ptr == 0 || cc == common->fast_forward_bc_ptr); +/* During recursion, these optimizations are disabled. */ +if (common->early_fail_start_ptr == 0) + { + early_fail_ptr = 0; + early_fail_type = type_skip; + } -if (cc == common->fast_forward_bc_ptr) - fast_fail = FALSE; -else if (common->fast_fail_start_ptr == 0) - fast_str_ptr = 0; +SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || early_fail_ptr == 0 + || (early_fail_ptr >= common->early_fail_start_ptr && early_fail_ptr <= common->early_fail_end_ptr)); -SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || fast_str_ptr == 0 - || (fast_str_ptr >= common->fast_fail_start_ptr && fast_str_ptr <= common->fast_fail_end_ptr)); +if (early_fail_type == type_fail) + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr)); cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); @@ -11345,13 +11257,11 @@ else tmp_offset = POSSESSIVE0; } -if (fast_fail && fast_str_ptr != 0) - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), fast_str_ptr)); - /* Handle fixed part first. */ if (exact > 1) { - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); + if (common->mode == PCRE2_JIT_COMPLETE #ifdef SUPPORT_UNICODE && !common->utf @@ -11376,18 +11286,31 @@ if (exact > 1) } } else if (exact == 1) + { compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); + if (early_fail_type == type_fail_range) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr + (int)sizeof(sljit_sw)); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, TMP2, 0); + OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, TMP2, 0); + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, TMP2, 0, TMP1, 0)); + + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr + (int)sizeof(sljit_sw), STR_PTR, 0); + } + } + switch(opcode) { case OP_STAR: case OP_UPTO: - SLJIT_ASSERT(fast_str_ptr == 0 || opcode == OP_STAR); + SLJIT_ASSERT(early_fail_ptr == 0 || opcode == OP_STAR); if (type == OP_ANYNL || type == OP_EXTUNI) { SLJIT_ASSERT(private_data_ptr == 0); - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); allocate_stack(common, 2); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); @@ -11406,180 +11329,231 @@ switch(opcode) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); } - /* We cannot use TMP3 because of this allocate_stack. */ + /* We cannot use TMP3 because of allocate_stack. */ allocate_stack(common, 1); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); JUMPTO(SLJIT_JUMP, label); if (jump != NULL) JUMPHERE(jump); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; } - else +#ifdef SUPPORT_UNICODE + else if (type == OP_ALLANY && !common->invalid_utf) +#else + else if (type == OP_ALLANY) +#endif { - charpos_enabled = FALSE; - charpos_char = 0; - charpos_othercasebit = 0; - - if ((type != OP_CHAR && type != OP_CHARI) && (*end == OP_CHAR || *end == OP_CHARI)) + if (opcode == OP_STAR) { - charpos_enabled = TRUE; + if (private_data_ptr == 0) + allocate_stack(common, 2); + + OP1(SLJIT_MOV, base, offset0, STR_END, 0); + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + + OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); + process_partial_match(common); + + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_END, 0); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; + } #ifdef SUPPORT_UNICODE - charpos_enabled = !common->utf || !HAS_EXTRALEN(end[1]); + else if (!common->utf) +#else + else #endif - if (charpos_enabled && *end == OP_CHARI && char_has_othercase(common, end + 1)) + { + if (private_data_ptr == 0) + allocate_stack(common, 2); + + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(max)); + + if (common->mode == PCRE2_JIT_COMPLETE) { - charpos_othercasebit = char_get_othercase_bit(common, end + 1); - if (charpos_othercasebit == 0) - charpos_enabled = FALSE; + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0); } - - if (charpos_enabled) + else { - charpos_char = end[1]; - /* Consumpe the OP_CHAR opcode. */ - end += 2; + jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, STR_END, 0); + process_partial_match(common); + JUMPHERE(jump); + } + + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; + } + } + + charpos_enabled = FALSE; + charpos_char = 0; + charpos_othercasebit = 0; + + if ((type != OP_CHAR && type != OP_CHARI) && (*end == OP_CHAR || *end == OP_CHARI)) + { +#ifdef SUPPORT_UNICODE + charpos_enabled = !common->utf || !HAS_EXTRALEN(end[1]); +#else + charpos_enabled = TRUE; +#endif + if (charpos_enabled && *end == OP_CHARI && char_has_othercase(common, end + 1)) + { + charpos_othercasebit = char_get_othercase_bit(common, end + 1); + if (charpos_othercasebit == 0) + charpos_enabled = FALSE; + } + + if (charpos_enabled) + { + charpos_char = end[1]; + /* Consume the OP_CHAR opcode. */ + end += 2; #if PCRE2_CODE_UNIT_WIDTH == 8 - SLJIT_ASSERT((charpos_othercasebit >> 8) == 0); + SLJIT_ASSERT((charpos_othercasebit >> 8) == 0); #elif PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 - SLJIT_ASSERT((charpos_othercasebit >> 9) == 0); - if ((charpos_othercasebit & 0x100) != 0) - charpos_othercasebit = (charpos_othercasebit & 0xff) << 8; + SLJIT_ASSERT((charpos_othercasebit >> 9) == 0); + if ((charpos_othercasebit & 0x100) != 0) + charpos_othercasebit = (charpos_othercasebit & 0xff) << 8; #endif - if (charpos_othercasebit != 0) - charpos_char |= charpos_othercasebit; + if (charpos_othercasebit != 0) + charpos_char |= charpos_othercasebit; - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.enabled = TRUE; - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.chr = charpos_char; - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.othercasebit = charpos_othercasebit; - } + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.enabled = TRUE; + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.chr = charpos_char; + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.othercasebit = charpos_othercasebit; } + } - if (charpos_enabled) - { - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max + 1); + if (charpos_enabled) + { + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max + 1); - /* Search the first instance of charpos_char. */ - jump = JUMP(SLJIT_JUMP); - label = LABEL(); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); - } - compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - JUMPHERE(jump); + /* Search the first instance of charpos_char. */ + jump = JUMP(SLJIT_JUMP); + label = LABEL(); + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); + } + compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); + JUMPHERE(jump); - detect_partial_match(common, &backtrack->topbacktracks); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (charpos_othercasebit != 0) - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); + detect_partial_match(common, &backtrack->topbacktracks); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (charpos_othercasebit != 0) + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); - if (private_data_ptr == 0) - allocate_stack(common, 2); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); - } + if (private_data_ptr == 0) + allocate_stack(common, 2); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - /* Search the last instance of charpos_char. */ - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, FALSE); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - detect_partial_match(common, &no_match); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (charpos_othercasebit != 0) - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); - if (opcode == OP_STAR) - { - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - } - else - { - jump = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - JUMPHERE(jump); - } + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); + } - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else - JUMPTO(SLJIT_JUMP, label); + /* Search the last instance of charpos_char. */ + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_match, FALSE); + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); + detect_partial_match(common, &no_match); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (charpos_othercasebit != 0) + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + if (opcode == OP_STAR) + { + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, label); } -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 - else if (common->utf) + else { - if (private_data_ptr == 0) - allocate_stack(common, 2); - + jump = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + JUMPHERE(jump); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + set_jumps(no_match, LABEL()); + OP2(SLJIT_ADD, STR_PTR, 0, base, offset0, SLJIT_IMM, IN_UCHARS(1)); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + } + else + { + if (private_data_ptr == 0) + allocate_stack(common, 2); - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + use_tmp = (!HAS_VIRTUAL_REGISTERS && opcode == OP_STAR); + SLJIT_ASSERT(!use_tmp || tmp_base == TMP3); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else - JUMPTO(SLJIT_JUMP, label); + if (common->utf) + OP1(SLJIT_MOV, use_tmp ? TMP3 : base, use_tmp ? 0 : offset0, STR_PTR, 0); +#endif + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - } + detect_partial_match(common, &no_match); + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf) + OP1(SLJIT_MOV, use_tmp ? TMP3 : base, use_tmp ? 0 : offset0, STR_PTR, 0); #endif - else + + if (opcode == OP_UPTO) { - if (private_data_ptr == 0) - allocate_stack(common, 2); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); + } - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + detect_partial_match_to(common, label); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - label = LABEL(); - detect_partial_match(common, &no_match); - compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - if (opcode == OP_UPTO) + set_jumps(no_char1_match, LABEL()); +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf) + { + set_jumps(no_match, LABEL()); + if (use_tmp) { - OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); + OP1(SLJIT_MOV, base, offset0, TMP3, 0); } else - JUMPTO(SLJIT_JUMP, label); - - set_jumps(no_char1_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + } + else +#endif + { OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); set_jumps(no_match, LABEL()); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); } + + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); } + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); break; @@ -11588,12 +11562,12 @@ switch(opcode) allocate_stack(common, 1); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); break; case OP_MINUPTO: - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); if (private_data_ptr == 0) allocate_stack(common, 2); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); @@ -11603,7 +11577,7 @@ switch(opcode) case OP_QUERY: case OP_MINQUERY: - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); if (private_data_ptr == 0) allocate_stack(common, 1); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); @@ -11616,63 +11590,112 @@ switch(opcode) break; case OP_POSSTAR: +#if defined SUPPORT_UNICODE + if (type == OP_ALLANY && !common->invalid_utf) +#else + if (type == OP_ALLANY) +#endif + { + OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); + process_partial_match(common); + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_END, 0); + break; + } + #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 if (common->utf) { OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); + detect_partial_match(common, &no_match); label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + compile_char1_matchingpath(common, type, cc, &no_match, FALSE); OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, label); + detect_partial_match_to(common, label); + set_jumps(no_match, LABEL()); OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + if (early_fail_ptr != 0) + { + if (!HAS_VIRTUAL_REGISTERS && tmp_base == TMP3) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, TMP3, 0); + else + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); + } break; } #endif - label = LABEL(); + detect_partial_match(common, &no_match); + label = LABEL(); compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - JUMPTO(SLJIT_JUMP, label); + detect_partial_match_to(common, label); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_char1_match, LABEL()); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); set_jumps(no_match, LABEL()); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + if (early_fail_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0); break; case OP_POSUPTO: - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 if (common->utf) { OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + + detect_partial_match(common, &no_match); label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + compile_char1_matchingpath(common, type, cc, &no_match, FALSE); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); + add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); + detect_partial_match_to(common, label); + set_jumps(no_match, LABEL()); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); break; } #endif + + if (type == OP_ALLANY) + { + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(max)); + + if (common->mode == PCRE2_JIT_COMPLETE) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0); + } + else + { + jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, STR_END, 0); + process_partial_match(common); + JUMPHERE(jump); + } + break; + } + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - label = LABEL(); + detect_partial_match(common, &no_match); + label = LABEL(); compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); + add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); + detect_partial_match_to(common, label); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_char1_match, LABEL()); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); set_jumps(no_match, LABEL()); break; case OP_POSQUERY: - SLJIT_ASSERT(fast_str_ptr == 0); + SLJIT_ASSERT(early_fail_ptr == 0); OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); compile_char1_matchingpath(common, type, cc, &no_match, TRUE); OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); @@ -11719,8 +11742,15 @@ if (common->accept_label == NULL) add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0))); else CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), common->accept_label); -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV_U32, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, options)); + +if (HAS_VIRTUAL_REGISTERS) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV_U32, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, options)); + } +else + OP1(SLJIT_MOV_U32, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, options)); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, PCRE2_NOTEMPTY); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_NOT_ZERO)); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, PCRE2_NOTEMPTY_ATSTART); @@ -11728,7 +11758,8 @@ if (common->accept_label == NULL) add_jump(compiler, &common->accept, JUMP(SLJIT_ZERO)); else JUMPTO(SLJIT_ZERO, common->accept_label); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(HAS_VIRTUAL_REGISTERS ? TMP1 : ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str)); if (common->accept_label == NULL) add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0)); else @@ -11778,10 +11809,11 @@ if (opcode == OP_SKIP) if (opcode == OP_COMMIT_ARG || opcode == OP_PRUNE_ARG || opcode == OP_THEN_ARG) { - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + if (HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(HAS_VIRTUAL_REGISTERS ? TMP1 : ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); } return ccend; @@ -12038,6 +12070,8 @@ while (cc < ccend) count_match(common); break; + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRA: @@ -12072,11 +12106,12 @@ while (cc < ccend) SLJIT_ASSERT(common->mark_ptr != 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); allocate_stack(common, common->has_skip_arg ? 5 : 1); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + if (HAS_VIRTUAL_REGISTERS) + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0), TMP2, 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(HAS_VIRTUAL_REGISTERS ? TMP1 : ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); if (common->has_skip_arg) { OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); @@ -12403,16 +12438,15 @@ PCRE2_SPTR ccprev; PCRE2_UCHAR bra = OP_BRA; PCRE2_UCHAR ket; assert_backtrack *assert; -sljit_uw *next_update_addr = NULL; BOOL has_alternatives; BOOL needs_control_head = FALSE; struct sljit_jump *brazero = NULL; -struct sljit_jump *alt1 = NULL; -struct sljit_jump *alt2 = NULL; +struct sljit_jump *next_alt = NULL; struct sljit_jump *once = NULL; struct sljit_jump *cond = NULL; struct sljit_label *rmin_label = NULL; struct sljit_label *exact_label = NULL; +struct sljit_put_label *put_label = NULL; if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) { @@ -12561,7 +12595,7 @@ else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND) free_stack(common, 1); alt_max = 2; - alt1 = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); + next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0); } } else if (has_alternatives) @@ -12569,21 +12603,16 @@ else if (has_alternatives) OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); free_stack(common, 1); - if (alt_max > 4) + if (alt_max > 3) { - /* Table jump if alt_max is greater than 4. */ - next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw)); - if (SLJIT_UNLIKELY(next_update_addr == NULL)) - return; - sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr); - add_label_addr(common, next_update_addr++); + sljit_emit_ijump(compiler, SLJIT_JUMP, TMP1, 0); + + SLJIT_ASSERT(CURRENT_AS(bracket_backtrack)->u.matching_put_label); + sljit_set_put_label(CURRENT_AS(bracket_backtrack)->u.matching_put_label, LABEL()); + sljit_emit_op0(compiler, SLJIT_ENDBR); } else - { - if (alt_max == 4) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); - } + next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0); } COMPILE_BACKTRACKINGPATH(current->top); @@ -12620,7 +12649,7 @@ if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) if (has_alternatives) { - alt_count = sizeof(sljit_uw); + alt_count = 1; do { current->top = NULL; @@ -12647,6 +12676,9 @@ if (has_alternatives) if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) return; + if (opcode == OP_ASSERT_NA || opcode == OP_ASSERTBACK_NA) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + if (opcode == OP_SCRIPT_RUN) match_script_run_common(common, private_data_ptr, current); } @@ -12699,7 +12731,12 @@ if (has_alternatives) stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); if (opcode != OP_ONCE) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, alt_count); + { + if (alt_max <= 3) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, alt_count); + else + put_label = sljit_emit_put_label(compiler, SLJIT_MEM1(STACK_TOP), STACK(stacksize)); + } if (offset != 0 && ket == OP_KETRMAX && common->optimized_cbracket[offset >> 1] != 0) { @@ -12712,24 +12749,21 @@ if (has_alternatives) if (opcode != OP_ONCE) { - if (alt_max > 4) - add_label_addr(common, next_update_addr++); - else + if (alt_max <= 3) { - if (alt_count != 2 * sizeof(sljit_uw)) + JUMPHERE(next_alt); + alt_count++; + if (alt_count < alt_max) { - JUMPHERE(alt1); - if (alt_max == 3 && alt_count == sizeof(sljit_uw)) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - } - else - { - JUMPHERE(alt2); - if (alt_max == 4) - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_uw)); + SLJIT_ASSERT(alt_count == 2 && alt_max == 3); + next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 1); } } - alt_count += sizeof(sljit_uw); + else + { + sljit_set_put_label(put_label, LABEL()); + sljit_emit_op0(compiler, SLJIT_ENDBR); + } } COMPILE_BACKTRACKINGPATH(current->top); @@ -12777,7 +12811,7 @@ if (offset != 0) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); } } -else if (opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_ASSERT_NA || opcode == OP_ASSERTBACK_NA || opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0)); free_stack(common, 1); @@ -13125,6 +13159,8 @@ while (current) compile_assert_backtrackingpath(common, current); break; + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_BRA: @@ -13219,11 +13255,10 @@ int private_data_size = get_recurse_data_length(common, ccbegin, ccend, &needs_c int alt_count, alt_max, local_size; backtrack_common altbacktrack; jump_list *match = NULL; -sljit_uw *next_update_addr = NULL; -struct sljit_jump *alt1 = NULL; -struct sljit_jump *alt2 = NULL; +struct sljit_jump *next_alt = NULL; struct sljit_jump *accept_exit = NULL; struct sljit_label *quit; +struct sljit_put_label *put_label = NULL; /* Recurse captures then. */ common->then_trap = NULL; @@ -13284,7 +13319,12 @@ while (1) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); if (alt_max > 1 || has_accept) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, alt_count); + { + if (alt_max > 3) + put_label = sljit_emit_put_label(compiler, SLJIT_MEM1(STACK_TOP), STACK(1)); + else + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, alt_count); + } add_jump(compiler, &match, JUMP(SLJIT_JUMP)); @@ -13298,7 +13338,7 @@ while (1) sljit_emit_fast_enter(compiler, TMP1, 0); if (has_accept) - accept_exit = CMP(SLJIT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, alt_max * sizeof (sljit_sw)); + accept_exit = CMP(SLJIT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, -1); OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); /* Save return address. */ @@ -13311,44 +13351,34 @@ while (1) OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); free_stack(common, 2); - if (alt_max > 4) + if (alt_max > 3) { - /* Table jump if alt_max is greater than 4. */ - next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw)); - if (SLJIT_UNLIKELY(next_update_addr == NULL)) - return; - sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr); - add_label_addr(common, next_update_addr++); + sljit_emit_ijump(compiler, SLJIT_JUMP, TMP1, 0); + sljit_set_put_label(put_label, LABEL()); + sljit_emit_op0(compiler, SLJIT_ENDBR); } else - { - if (alt_max == 4) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); - } + next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0); } else free_stack(common, has_accept ? 2 : 1); } - else if (alt_max > 4) - add_label_addr(common, next_update_addr++); + else if (alt_max > 3) + { + sljit_set_put_label(put_label, LABEL()); + sljit_emit_op0(compiler, SLJIT_ENDBR); + } else { - if (alt_count != 2 * sizeof(sljit_uw)) + JUMPHERE(next_alt); + if (alt_count + 1 < alt_max) { - JUMPHERE(alt1); - if (alt_max == 3 && alt_count == sizeof(sljit_uw)) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - } - else - { - JUMPHERE(alt2); - if (alt_max == 4) - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_uw)); + SLJIT_ASSERT(alt_count == 1 && alt_max == 3); + next_alt = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 1); } } - alt_count += sizeof(sljit_uw); + alt_count++; compile_backtrackingpath(common, altbacktrack.top); if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) @@ -13371,7 +13401,7 @@ copy_recurse_data(common, ccbegin, ccend, recurse_copy_private_to_global, local_ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1)); free_stack(common, private_data_size + local_size); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -sljit_emit_fast_return(compiler, TMP2, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP2, 0); if (common->quit != NULL) { @@ -13396,7 +13426,7 @@ if (has_accept) OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(local_size - 1)); free_stack(common, private_data_size + local_size); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); - sljit_emit_fast_return(compiler, TMP2, 0); + OP_SRC(SLJIT_FAST_RETURN, TMP2, 0); } if (common->accept != NULL) @@ -13409,7 +13439,7 @@ if (common->accept != NULL) OP1(SLJIT_MOV, TMP2, 0, STACK_TOP, 0); allocate_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, alt_count); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, -1); } set_jumps(match, LABEL()); @@ -13420,7 +13450,7 @@ copy_recurse_data(common, ccbegin, ccend, recurse_swap_global, local_size, priva OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), STACK(local_size - 1)); OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); -sljit_emit_fast_return(compiler, TMP2, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP2, 0); } #undef COMPILE_BACKTRACKINGPATH @@ -13444,7 +13474,6 @@ executable_functions *functions; void *executable_func; sljit_uw executable_size; sljit_uw total_length; -label_addr_list *label_addr; struct sljit_label *mainloop_label = NULL; struct sljit_label *continue_match_label; struct sljit_label *empty_match_found_label = NULL; @@ -13453,12 +13482,20 @@ struct sljit_label *reset_match_label; struct sljit_label *quit_label; struct sljit_jump *jump; struct sljit_jump *minlength_check_failed = NULL; -struct sljit_jump *reqbyte_notfound = NULL; struct sljit_jump *empty_match = NULL; struct sljit_jump *end_anchor_failed = NULL; +jump_list *reqcu_not_found = NULL; SLJIT_ASSERT(tables); +#if HAS_VIRTUAL_REGISTERS == 1 +SLJIT_ASSERT(sljit_get_register_index(TMP3) < 0 && sljit_get_register_index(ARGUMENTS) < 0 && sljit_get_register_index(RETURN_ADDR) < 0); +#elif HAS_VIRTUAL_REGISTERS == 0 +SLJIT_ASSERT(sljit_get_register_index(TMP3) >= 0 && sljit_get_register_index(ARGUMENTS) >= 0 && sljit_get_register_index(RETURN_ADDR) >= 0); +#else +#error "Invalid value for HAS_VIRTUAL_REGISTERS" +#endif + memset(&rootbacktrack, 0, sizeof(backtrack_common)); memset(common, 0, sizeof(compiler_common)); common->re = re; @@ -13475,7 +13512,8 @@ common->read_only_data_head = NULL; common->fcc = tables + fcc_offset; common->lcc = (sljit_sw)(tables + lcc_offset); common->mode = mode; -common->might_be_empty = re->minlength == 0; +common->might_be_empty = (re->minlength == 0) || (re->flags & PCRE2_MATCH_EMPTY); +common->allow_empty_partial = (re->max_lookbehind > 0) || (re->flags & PCRE2_MATCH_EMPTY); common->nltype = NLTYPE_FIXED; switch(re->newline_convention) { @@ -13512,7 +13550,7 @@ common->alt_circumflex = (re->overall_options & PCRE2_ALT_CIRCUMFLEX) != 0; #ifdef SUPPORT_UNICODE /* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ common->utf = (re->overall_options & PCRE2_UTF) != 0; -common->use_ucp = (re->overall_options & PCRE2_UCP) != 0; +common->ucp = (re->overall_options & PCRE2_UCP) != 0; if (common->utf) { if (common->nltype == NLTYPE_ANY) @@ -13624,13 +13662,10 @@ memset(common->private_data_ptrs, 0, total_length * sizeof(sljit_s32)); private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw); set_private_data_ptrs(common, &private_data_size, ccend); -if ((re->overall_options & PCRE2_ANCHORED) == 0 && (re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0) - { - if (!detect_fast_forward_skip(common, &private_data_size) && !common->has_skip_in_assert_back) - detect_fast_fail(common, common->start, &private_data_size, 4); - } +if ((re->overall_options & PCRE2_ANCHORED) == 0 && (re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0 && !common->has_skip_in_assert_back) + detect_early_fail(common, common->start, &private_data_size, 0, 0); -SLJIT_ASSERT(common->fast_fail_start_ptr <= common->fast_fail_end_ptr); +SLJIT_ASSERT(common->early_fail_start_ptr <= common->early_fail_end_ptr); if (private_data_size > SLJIT_MAX_LOCAL_SIZE) { @@ -13646,7 +13681,7 @@ if (common->has_then) set_then_offsets(common, common->start, NULL); } -compiler = sljit_create_compiler(allocator_data); +compiler = sljit_create_compiler(allocator_data, NULL); if (!compiler) { SLJIT_FREE(common->optimized_cbracket, allocator_data); @@ -13674,8 +13709,8 @@ OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_sta OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0); -if (common->fast_fail_start_ptr < common->fast_fail_end_ptr) - reset_fast_fail(common); +if (common->early_fail_start_ptr < common->early_fail_end_ptr) + reset_early_fail(common); if (mode == PCRE2_JIT_PARTIAL_SOFT) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); @@ -13712,7 +13747,7 @@ if (mode == PCRE2_JIT_COMPLETE && re->minlength > 0 && (re->overall_options & PC minlength_check_failed = CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0); } if (common->req_char_ptr != 0) - reqbyte_notfound = search_requested_char(common, (PCRE2_UCHAR)(re->last_codeunit), (re->flags & PCRE2_LASTCASELESS) != 0, (re->flags & PCRE2_FIRSTSET) != 0); + reqcu_not_found = search_requested_char(common, (PCRE2_UCHAR)(re->last_codeunit), (re->flags & PCRE2_LASTCASELESS) != 0, (re->flags & PCRE2_FIRSTSET) != 0); /* Store the current STR_PTR in OVECTOR(0). */ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0); @@ -13721,7 +13756,7 @@ OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH); if (common->capture_last_ptr != 0) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, 0); if (common->fast_forward_bc_ptr != NULL) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1) >> 3, STR_PTR, 0); if (common->start_ptr != OVECTOR(0)) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0); @@ -13742,7 +13777,7 @@ if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) sljit_free_compiler(compiler); SLJIT_FREE(common->optimized_cbracket, allocator_data); SLJIT_FREE(common->private_data_ptrs, allocator_data); - PRIV(jit_free_rodata)(common->read_only_data_head, compiler->allocator_data); + PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } @@ -13768,6 +13803,8 @@ if (common->abort != NULL) set_jumps(common->abort, common->abort_label); if (minlength_check_failed != NULL) SET_LABEL(minlength_check_failed, common->abort_label); + +sljit_emit_op0(compiler, SLJIT_SKIP_FRAMES_BEFORE_RETURN); sljit_emit_return(compiler, SLJIT_MOV, SLJIT_RETURN_REG, 0); if (common->failed_match != NULL) @@ -13796,7 +13833,7 @@ if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) sljit_free_compiler(compiler); SLJIT_FREE(common->optimized_cbracket, allocator_data); SLJIT_FREE(common->private_data_ptrs, allocator_data); - PRIV(jit_free_rodata)(common->read_only_data_head, compiler->allocator_data); + PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } @@ -13820,7 +13857,7 @@ if ((re->overall_options & PCRE2_ANCHORED) == 0 && common->match_end_ptr != 0) } OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), - (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1)) : common->start_ptr); + (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1) >> 3) : common->start_ptr); if ((re->overall_options & PCRE2_ANCHORED) == 0) { @@ -13845,8 +13882,8 @@ if ((re->overall_options & PCRE2_ANCHORED) == 0) } /* No more remaining characters. */ -if (reqbyte_notfound != NULL) - JUMPHERE(reqbyte_notfound); +if (reqcu_not_found != NULL) + set_jumps(reqcu_not_found, LABEL()); if (mode == PCRE2_JIT_PARTIAL_SOFT) CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1, common->partialmatchlabel); @@ -13871,8 +13908,8 @@ if (common->might_be_empty) } common->fast_forward_bc_ptr = NULL; -common->fast_fail_start_ptr = 0; -common->fast_fail_end_ptr = 0; +common->early_fail_start_ptr = 0; +common->early_fail_end_ptr = 0; common->currententry = common->entries; common->local_quit_available = TRUE; quit_label = common->quit_label; @@ -13885,7 +13922,7 @@ while (common->currententry != NULL) sljit_free_compiler(compiler); SLJIT_FREE(common->optimized_cbracket, allocator_data); SLJIT_FREE(common->private_data_ptrs, allocator_data); - PRIV(jit_free_rodata)(common->read_only_data_head, compiler->allocator_data); + PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } flush_stubs(common); @@ -13915,7 +13952,7 @@ OP1(SLJIT_MOV, TMP2, 0, STACK_LIMIT, 0); OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_RETURN_REG, 0); OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, TMP1, 0); +OP_SRC(SLJIT_FAST_RETURN, TMP1, 0); /* Allocation failed. */ JUMPHERE(jump); @@ -14028,16 +14065,11 @@ SLJIT_FREE(common->private_data_ptrs, allocator_data); executable_func = sljit_generate_code(compiler); executable_size = sljit_get_generated_code_size(compiler); -label_addr = common->label_addrs; -while (label_addr != NULL) - { - *label_addr->update_addr = sljit_get_label_addr(label_addr->label); - label_addr = label_addr->next; - } sljit_free_compiler(compiler); + if (executable_func == NULL) { - PRIV(jit_free_rodata)(common->read_only_data_head, compiler->allocator_data); + PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } @@ -14051,8 +14083,8 @@ else { /* This case is highly unlikely since we just recently freed a lot of memory. Not impossible though. */ - sljit_free_code(executable_func); - PRIV(jit_free_rodata)(common->read_only_data_head, compiler->allocator_data); + sljit_free_code(executable_func, NULL); + PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } memset(functions, 0, sizeof(executable_functions)); @@ -14097,18 +14129,7 @@ Returns: 0: success or (*NOJIT) was used PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION pcre2_jit_compile(pcre2_code *code, uint32_t options) { -#ifndef SUPPORT_JIT - -(void)code; -(void)options; -return PCRE2_ERROR_JIT_BADOPTION; - -#else /* SUPPORT_JIT */ - pcre2_real_code *re = (pcre2_real_code *)code; -executable_functions *functions; -uint32_t excluded_options; -int result; if (code == NULL) return PCRE2_ERROR_NULL; @@ -14116,30 +14137,103 @@ if (code == NULL) if ((options & ~PUBLIC_JIT_COMPILE_OPTIONS) != 0) return PCRE2_ERROR_JIT_BADOPTION; +/* Support for invalid UTF was first introduced in JIT, with the option +PCRE2_JIT_INVALID_UTF. Later, support was added to the interpreter, and the +compile-time option PCRE2_MATCH_INVALID_UTF was created. This is now the +preferred feature, with the earlier option deprecated. However, for backward +compatibility, if the earlier option is set, it forces the new option so that +if JIT matching falls back to the interpreter, there is still support for +invalid UTF. However, if this function has already been successfully called +without PCRE2_JIT_INVALID_UTF and without PCRE2_MATCH_INVALID_UTF (meaning that +non-invalid-supporting JIT code was compiled), give an error. + +If in the future support for PCRE2_JIT_INVALID_UTF is withdrawn, the following +actions are needed: + + 1. Remove the definition from pcre2.h.in and from the list in + PUBLIC_JIT_COMPILE_OPTIONS above. + + 2. Replace PCRE2_JIT_INVALID_UTF with a local flag in this module. + + 3. Replace PCRE2_JIT_INVALID_UTF in pcre2_jit_test.c. + + 4. Delete the following short block of code. The setting of "re" and + "functions" can be moved into the JIT-only block below, but if that is + done, (void)re and (void)functions will be needed in the non-JIT case, to + avoid compiler warnings. +*/ + +#ifdef SUPPORT_JIT +executable_functions *functions = (executable_functions *)re->executable_jit; +static int executable_allocator_is_working = 0; +#endif + +if ((options & PCRE2_JIT_INVALID_UTF) != 0) + { + if ((re->overall_options & PCRE2_MATCH_INVALID_UTF) == 0) + { +#ifdef SUPPORT_JIT + if (functions != NULL) return PCRE2_ERROR_JIT_BADOPTION; +#endif + re->overall_options |= PCRE2_MATCH_INVALID_UTF; + } + } + +/* The above tests are run with and without JIT support. This means that +PCRE2_JIT_INVALID_UTF propagates back into the regex options (ensuring +interpreter support) even in the absence of JIT. But now, if there is no JIT +support, give an error return. */ + +#ifndef SUPPORT_JIT +return PCRE2_ERROR_JIT_BADOPTION; +#else /* SUPPORT_JIT */ + +/* There is JIT support. Do the necessary. */ + if ((re->flags & PCRE2_NOJIT) != 0) return 0; -functions = (executable_functions *)re->executable_jit; +if (executable_allocator_is_working == 0) + { + /* Checks whether the executable allocator is working. This check + might run multiple times in multi-threaded environments, but the + result should not be affected by it. */ + void *ptr = SLJIT_MALLOC_EXEC(32, NULL); + + executable_allocator_is_working = -1; + + if (ptr != NULL) + { + SLJIT_FREE_EXEC(((sljit_u8*)(ptr)) + SLJIT_EXEC_OFFSET(ptr), NULL); + executable_allocator_is_working = 1; + } + } + +if (executable_allocator_is_working < 0) + return PCRE2_ERROR_NOMEMORY; + +if ((re->overall_options & PCRE2_MATCH_INVALID_UTF) != 0) + options |= PCRE2_JIT_INVALID_UTF; if ((options & PCRE2_JIT_COMPLETE) != 0 && (functions == NULL || functions->executable_funcs[0] == NULL)) { - excluded_options = (PCRE2_JIT_PARTIAL_SOFT | PCRE2_JIT_PARTIAL_HARD); - result = jit_compile(code, options & ~excluded_options); + uint32_t excluded_options = (PCRE2_JIT_PARTIAL_SOFT | PCRE2_JIT_PARTIAL_HARD); + int result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } if ((options & PCRE2_JIT_PARTIAL_SOFT) != 0 && (functions == NULL || functions->executable_funcs[1] == NULL)) { - excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_HARD); - result = jit_compile(code, options & ~excluded_options); + uint32_t excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_HARD); + int result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } if ((options & PCRE2_JIT_PARTIAL_HARD) != 0 && (functions == NULL || functions->executable_funcs[2] == NULL)) { - excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_SOFT); - result = jit_compile(code, options & ~excluded_options); + uint32_t excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_SOFT); + int result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } diff --git a/dist2/src/pcre2_jit_match.c b/dist2/src/pcre2_jit_match.c index eee03864..7e13b8cf 100644 --- a/dist2/src/pcre2_jit_match.c +++ b/dist2/src/pcre2_jit_match.c @@ -74,7 +74,6 @@ Arguments: options option bits match_data points to a match_data block mcontext points to a match context - jit_stack points to a JIT stack Returns: > 0 => success; value is the number of ovector pairs filled = 0 => success, but ovector is not big enough diff --git a/dist2/src/pcre2_jit_misc.c b/dist2/src/pcre2_jit_misc.c index efdb0558..ec924e0f 100644 --- a/dist2/src/pcre2_jit_misc.c +++ b/dist2/src/pcre2_jit_misc.c @@ -89,7 +89,7 @@ int i; for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) { if (functions->executable_funcs[i] != NULL) - sljit_free_code(functions->executable_funcs[i]); + sljit_free_code(functions->executable_funcs[i], NULL); PRIV(jit_free_rodata)(functions->read_only_data_heads[i], allocator_data); } @@ -145,6 +145,11 @@ maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1); jit_stack = PRIV(memctl_malloc)(sizeof(pcre2_real_jit_stack), (pcre2_memctl *)gcontext); if (jit_stack == NULL) return NULL; jit_stack->stack = sljit_allocate_stack(startsize, maxsize, &jit_stack->memctl); +if (jit_stack->stack == NULL) + { + jit_stack->memctl.free(jit_stack, jit_stack->memctl.memory_data); + return NULL; + } return jit_stack; #endif diff --git a/dist2/src/pcre2_jit_neon_inc.h b/dist2/src/pcre2_jit_neon_inc.h new file mode 100644 index 00000000..150da29e --- /dev/null +++ b/dist2/src/pcre2_jit_neon_inc.h @@ -0,0 +1,347 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + This module by Zoltan Herczeg and Sebastian Pop + Original API code Copyright (c) 1997-2012 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +# if defined(FFCS) +# if defined(FF_UTF) +# define FF_FUN ffcs_utf +# else +# define FF_FUN ffcs +# endif + +# elif defined(FFCS_2) +# if defined(FF_UTF) +# define FF_FUN ffcs_2_utf +# else +# define FF_FUN ffcs_2 +# endif + +# elif defined(FFCS_MASK) +# if defined(FF_UTF) +# define FF_FUN ffcs_mask_utf +# else +# define FF_FUN ffcs_mask +# endif + +# elif defined(FFCPS_0) +# if defined (FF_UTF) +# define FF_FUN ffcps_0_utf +# else +# define FF_FUN ffcps_0 +# endif + +# elif defined (FFCPS_1) +# if defined (FF_UTF) +# define FF_FUN ffcps_1_utf +# else +# define FF_FUN ffcps_1 +# endif + +# elif defined (FFCPS_DEFAULT) +# if defined (FF_UTF) +# define FF_FUN ffcps_default_utf +# else +# define FF_FUN ffcps_default +# endif +# endif + +static sljit_u8* SLJIT_FUNC FF_FUN(sljit_u8 *str_end, sljit_u8 *str_ptr, sljit_uw offs1, sljit_uw offs2, sljit_uw chars) +#undef FF_FUN +{ +quad_word qw; +int_char ic; + +SLJIT_UNUSED_ARG(offs1); +SLJIT_UNUSED_ARG(offs2); + +ic.x = chars; + +#if defined(FFCS) +sljit_u8 c1 = ic.c.c1; +vect_t vc1 = VDUPQ(c1); + +#elif defined(FFCS_2) +sljit_u8 c1 = ic.c.c1; +vect_t vc1 = VDUPQ(c1); +sljit_u8 c2 = ic.c.c2; +vect_t vc2 = VDUPQ(c2); + +#elif defined(FFCS_MASK) +sljit_u8 c1 = ic.c.c1; +vect_t vc1 = VDUPQ(c1); +sljit_u8 mask = ic.c.c2; +vect_t vmask = VDUPQ(mask); +#endif + +#if defined(FFCPS) +compare_type compare1_type = compare_match1; +compare_type compare2_type = compare_match1; +vect_t cmp1a, cmp1b, cmp2a, cmp2b; +const sljit_u32 diff = IN_UCHARS(offs1 - offs2); +PCRE2_UCHAR char1a = ic.c.c1; +PCRE2_UCHAR char2a = ic.c.c3; + +# ifdef FFCPS_CHAR1A2A +cmp1a = VDUPQ(char1a); +cmp2a = VDUPQ(char2a); +cmp1b = VDUPQ(0); /* to avoid errors on older compilers -Werror=maybe-uninitialized */ +cmp2b = VDUPQ(0); /* to avoid errors on older compilers -Werror=maybe-uninitialized */ +# else +PCRE2_UCHAR char1b = ic.c.c2; +PCRE2_UCHAR char2b = ic.c.c4; +if (char1a == char1b) + { + cmp1a = VDUPQ(char1a); + cmp1b = VDUPQ(0); /* to avoid errors on older compilers -Werror=maybe-uninitialized */ + } +else + { + sljit_u32 bit1 = char1a ^ char1b; + if (is_powerof2(bit1)) + { + compare1_type = compare_match1i; + cmp1a = VDUPQ(char1a | bit1); + cmp1b = VDUPQ(bit1); + } + else + { + compare1_type = compare_match2; + cmp1a = VDUPQ(char1a); + cmp1b = VDUPQ(char1b); + } + } + +if (char2a == char2b) + { + cmp2a = VDUPQ(char2a); + cmp2b = VDUPQ(0); /* to avoid errors on older compilers -Werror=maybe-uninitialized */ + } +else + { + sljit_u32 bit2 = char2a ^ char2b; + if (is_powerof2(bit2)) + { + compare2_type = compare_match1i; + cmp2a = VDUPQ(char2a | bit2); + cmp2b = VDUPQ(bit2); + } + else + { + compare2_type = compare_match2; + cmp2a = VDUPQ(char2a); + cmp2b = VDUPQ(char2b); + } + } +# endif + +str_ptr += IN_UCHARS(offs1); +#endif + +#if PCRE2_CODE_UNIT_WIDTH != 8 +vect_t char_mask = VDUPQ(0xff); +#endif + +#if defined(FF_UTF) +restart:; +#endif + +#if defined(FFCPS) +sljit_u8 *p1 = str_ptr - diff; +#endif +sljit_s32 align_offset = ((uint64_t)str_ptr & 0xf); +str_ptr = (sljit_u8 *) ((uint64_t)str_ptr & ~0xf); +vect_t data = VLD1Q(str_ptr); +#if PCRE2_CODE_UNIT_WIDTH != 8 +data = VANDQ(data, char_mask); +#endif + +#if defined(FFCS) +vect_t eq = VCEQQ(data, vc1); + +#elif defined(FFCS_2) +vect_t eq1 = VCEQQ(data, vc1); +vect_t eq2 = VCEQQ(data, vc2); +vect_t eq = VORRQ(eq1, eq2); + +#elif defined(FFCS_MASK) +vect_t eq = VORRQ(data, vmask); +eq = VCEQQ(eq, vc1); + +#elif defined(FFCPS) +# if defined(FFCPS_DIFF1) +vect_t prev_data = data; +# endif + +vect_t data2; +if (p1 < str_ptr) + { + data2 = VLD1Q(str_ptr - diff); +#if PCRE2_CODE_UNIT_WIDTH != 8 + data2 = VANDQ(data2, char_mask); +#endif + } +else + data2 = shift_left_n_lanes(data, offs1 - offs2); + +if (compare1_type == compare_match1) + data = VCEQQ(data, cmp1a); +else + data = fast_forward_char_pair_compare(compare1_type, data, cmp1a, cmp1b); + +if (compare2_type == compare_match1) + data2 = VCEQQ(data2, cmp2a); +else + data2 = fast_forward_char_pair_compare(compare2_type, data2, cmp2a, cmp2b); + +vect_t eq = VANDQ(data, data2); +#endif + +VST1Q(qw.mem, eq); +/* Ignore matches before the first STR_PTR. */ +if (align_offset < 8) + { + qw.dw[0] >>= align_offset * 8; + if (qw.dw[0]) + { + str_ptr += align_offset + __builtin_ctzll(qw.dw[0]) / 8; + goto match; + } + if (qw.dw[1]) + { + str_ptr += 8 + __builtin_ctzll(qw.dw[1]) / 8; + goto match; + } + } +else + { + qw.dw[1] >>= (align_offset - 8) * 8; + if (qw.dw[1]) + { + str_ptr += align_offset + __builtin_ctzll(qw.dw[1]) / 8; + goto match; + } + } +str_ptr += 16; + +while (str_ptr < str_end) + { + vect_t orig_data = VLD1Q(str_ptr); +#if PCRE2_CODE_UNIT_WIDTH != 8 + orig_data = VANDQ(orig_data, char_mask); +#endif + data = orig_data; + +#if defined(FFCS) + eq = VCEQQ(data, vc1); + +#elif defined(FFCS_2) + eq1 = VCEQQ(data, vc1); + eq2 = VCEQQ(data, vc2); + eq = VORRQ(eq1, eq2); + +#elif defined(FFCS_MASK) + eq = VORRQ(data, vmask); + eq = VCEQQ(eq, vc1); +#endif + +#if defined(FFCPS) +# if defined (FFCPS_DIFF1) + data2 = VEXTQ(prev_data, data, VECTOR_FACTOR - 1); +# else + data2 = VLD1Q(str_ptr - diff); +# if PCRE2_CODE_UNIT_WIDTH != 8 + data2 = VANDQ(data2, char_mask); +# endif +# endif + +# ifdef FFCPS_CHAR1A2A + data = VCEQQ(data, cmp1a); + data2 = VCEQQ(data2, cmp2a); +# else + if (compare1_type == compare_match1) + data = VCEQQ(data, cmp1a); + else + data = fast_forward_char_pair_compare(compare1_type, data, cmp1a, cmp1b); + if (compare2_type == compare_match1) + data2 = VCEQQ(data2, cmp2a); + else + data2 = fast_forward_char_pair_compare(compare2_type, data2, cmp2a, cmp2b); +# endif + + eq = VANDQ(data, data2); +#endif + + VST1Q(qw.mem, eq); + if (qw.dw[0]) + str_ptr += __builtin_ctzll(qw.dw[0]) / 8; + else if (qw.dw[1]) + str_ptr += 8 + __builtin_ctzll(qw.dw[1]) / 8; + else { + str_ptr += 16; +#if defined (FFCPS_DIFF1) + prev_data = orig_data; +#endif + continue; + } + +match:; + if (str_ptr >= str_end) + /* Failed match. */ + return NULL; + +#if defined(FF_UTF) + if (utf_continue(str_ptr + IN_UCHARS(-offs1))) + { + /* Not a match. */ + str_ptr += IN_UCHARS(1); + goto restart; + } +#endif + + /* Match. */ +#if defined (FFCPS) + str_ptr -= IN_UCHARS(offs1); +#endif + return str_ptr; + } + +/* Failed match. */ +return NULL; +} diff --git a/dist2/src/pcre2_jit_simd_inc.h b/dist2/src/pcre2_jit_simd_inc.h new file mode 100644 index 00000000..5673d338 --- /dev/null +++ b/dist2/src/pcre2_jit_simd_inc.h @@ -0,0 +1,1123 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + This module by Zoltan Herczeg + Original API code Copyright (c) 1997-2012 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +static struct sljit_jump *jump_if_utf_char_start(struct sljit_compiler *compiler, sljit_s32 reg) +{ +#if PCRE2_CODE_UNIT_WIDTH == 8 +OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xc0); +return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0x80); +#elif PCRE2_CODE_UNIT_WIDTH == 16 +OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xfc00); +return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0xdc00); +#else +#error "Unknown code width" +#endif +} +#endif + +static sljit_s32 character_to_int32(PCRE2_UCHAR chr) +{ +sljit_u32 value = chr; +#if PCRE2_CODE_UNIT_WIDTH == 8 +#define SSE2_COMPARE_TYPE_INDEX 0 +return (sljit_s32)((value << 24) | (value << 16) | (value << 8) | value); +#elif PCRE2_CODE_UNIT_WIDTH == 16 +#define SSE2_COMPARE_TYPE_INDEX 1 +return (sljit_s32)((value << 16) | value); +#elif PCRE2_CODE_UNIT_WIDTH == 32 +#define SSE2_COMPARE_TYPE_INDEX 2 +return (sljit_s32)(value); +#else +#error "Unsupported unit width" +#endif +} + +static void load_from_mem_sse2(struct sljit_compiler *compiler, sljit_s32 dst_xmm_reg, sljit_s32 src_general_reg, sljit_s8 offset) +{ +sljit_u8 instruction[5]; + +SLJIT_ASSERT(dst_xmm_reg < 8); +SLJIT_ASSERT(src_general_reg < 8); + +/* MOVDQA xmm1, xmm2/m128 */ +instruction[0] = ((sljit_u8)offset & 0xf) == 0 ? 0x66 : 0xf3; +instruction[1] = 0x0f; +instruction[2] = 0x6f; + +if (offset == 0) + { + instruction[3] = (dst_xmm_reg << 3) | src_general_reg; + sljit_emit_op_custom(compiler, instruction, 4); + return; + } + +instruction[3] = 0x40 | (dst_xmm_reg << 3) | src_general_reg; +instruction[4] = (sljit_u8)offset; +sljit_emit_op_custom(compiler, instruction, 5); +} + +typedef enum { + sse2_compare_match1, + sse2_compare_match1i, + sse2_compare_match2, +} sse2_compare_type; + +static void fast_forward_char_pair_sse2_compare(struct sljit_compiler *compiler, sse2_compare_type compare_type, + int step, sljit_s32 dst_ind, sljit_s32 cmp1_ind, sljit_s32 cmp2_ind, sljit_s32 tmp_ind) +{ +sljit_u8 instruction[4]; +instruction[0] = 0x66; +instruction[1] = 0x0f; + +SLJIT_ASSERT(step >= 0 && step <= 3); + +if (compare_type != sse2_compare_match2) + { + if (step == 0) + { + if (compare_type == sse2_compare_match1i) + { + /* POR xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0xeb; + instruction[3] = 0xc0 | (dst_ind << 3) | cmp2_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + return; + } + + if (step != 2) + return; + + /* PCMPEQB/W/D xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; + instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind; + sljit_emit_op_custom(compiler, instruction, 4); + return; + } + +switch (step) + { + case 0: + /* MOVDQA xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0x6f; + instruction[3] = 0xc0 | (tmp_ind << 3) | dst_ind; + sljit_emit_op_custom(compiler, instruction, 4); + return; + + case 1: + /* PCMPEQB/W/D xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; + instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind; + sljit_emit_op_custom(compiler, instruction, 4); + return; + + case 2: + /* PCMPEQB/W/D xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; + instruction[3] = 0xc0 | (tmp_ind << 3) | cmp2_ind; + sljit_emit_op_custom(compiler, instruction, 4); + return; + + case 3: + /* POR xmm1, xmm2/m128 */ + /* instruction[0] = 0x66; */ + /* instruction[1] = 0x0f; */ + instruction[2] = 0xeb; + instruction[3] = 0xc0 | (dst_ind << 3) | tmp_ind; + sljit_emit_op_custom(compiler, instruction, 4); + return; + } +} + +#define JIT_HAS_FAST_FORWARD_CHAR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) + +static void fast_forward_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset) +{ +DEFINE_COMPILER; +struct sljit_label *start; +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +struct sljit_label *restart; +#endif +struct sljit_jump *quit; +struct sljit_jump *partial_quit[2]; +sse2_compare_type compare_type = sse2_compare_match1; +sljit_u8 instruction[8]; +sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1); +sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR); +sljit_s32 data_ind = 0; +sljit_s32 tmp_ind = 1; +sljit_s32 cmp1_ind = 2; +sljit_s32 cmp2_ind = 3; +sljit_u32 bit = 0; +int i; + +SLJIT_UNUSED_ARG(offset); + +if (char1 != char2) + { + bit = char1 ^ char2; + compare_type = sse2_compare_match1i; + + if (!is_powerof2(bit)) + { + bit = 0; + compare_type = sse2_compare_match2; + } + } + +partial_quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +if (common->mode == PCRE2_JIT_COMPLETE) + add_jump(compiler, &common->failed_match, partial_quit[0]); + +/* First part (unaligned start) */ + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit)); + +SLJIT_ASSERT(tmp1_reg_ind < 8); + +/* MOVD xmm, r/m32 */ +instruction[0] = 0x66; +instruction[1] = 0x0f; +instruction[2] = 0x6e; +instruction[3] = 0xc0 | (cmp1_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (char1 != char2) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2)); + + /* MOVD xmm, r/m32 */ + instruction[3] = 0xc0 | (cmp2_ind << 3) | tmp1_reg_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); + +/* PSHUFD xmm1, xmm2/m128, imm8 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0x70; +instruction[3] = 0xc0 | (cmp1_ind << 3) | cmp1_ind; +instruction[4] = 0; +sljit_emit_op_custom(compiler, instruction, 5); + +if (char1 != char2) + { + /* PSHUFD xmm1, xmm2/m128, imm8 */ + instruction[3] = 0xc0 | (cmp2_ind << 3) | cmp2_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +restart = LABEL(); +#endif +OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf); + +load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0); +for (i = 0; i < 4; i++) + fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0); + +quit = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + +/* Second part (aligned) */ +start = LABEL(); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); + +partial_quit[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +if (common->mode == PCRE2_JIT_COMPLETE) + add_jump(compiler, &common->failed_match, partial_quit[1]); + +load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0); +for (i = 0; i < 4; i++) + fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start); + +JUMPHERE(quit); + +/* BSF r32, r/m32 */ +instruction[0] = 0x0f; +instruction[1] = 0xbc; +instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 3); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + +if (common->mode != PCRE2_JIT_COMPLETE) + { + JUMPHERE(partial_quit[0]); + JUMPHERE(partial_quit[1]); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0); + } +else + add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +if (common->utf && offset > 0) + { + SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE); + + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); + + quit = jump_if_utf_char_start(compiler, TMP1); + + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, restart); + + JUMPHERE(quit); + } +#endif +} + +#define JIT_HAS_FAST_REQUESTED_CHAR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) + +static jump_list *fast_requested_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2) +{ +DEFINE_COMPILER; +struct sljit_label *start; +struct sljit_jump *quit; +jump_list *not_found = NULL; +sse2_compare_type compare_type = sse2_compare_match1; +sljit_u8 instruction[8]; +sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1); +sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR); +sljit_s32 data_ind = 0; +sljit_s32 tmp_ind = 1; +sljit_s32 cmp1_ind = 2; +sljit_s32 cmp2_ind = 3; +sljit_u32 bit = 0; +int i; + +if (char1 != char2) + { + bit = char1 ^ char2; + compare_type = sse2_compare_match1i; + + if (!is_powerof2(bit)) + { + bit = 0; + compare_type = sse2_compare_match2; + } + } + +add_jump(compiler, ¬_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0)); +OP1(SLJIT_MOV, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + +/* First part (unaligned start) */ + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit)); + +SLJIT_ASSERT(tmp1_reg_ind < 8); + +/* MOVD xmm, r/m32 */ +instruction[0] = 0x66; +instruction[1] = 0x0f; +instruction[2] = 0x6e; +instruction[3] = 0xc0 | (cmp1_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (char1 != char2) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2)); + + /* MOVD xmm, r/m32 */ + instruction[3] = 0xc0 | (cmp2_ind << 3) | tmp1_reg_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +OP1(SLJIT_MOV, STR_PTR, 0, TMP2, 0); + +/* PSHUFD xmm1, xmm2/m128, imm8 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0x70; +instruction[3] = 0xc0 | (cmp1_ind << 3) | cmp1_ind; +instruction[4] = 0; +sljit_emit_op_custom(compiler, instruction, 5); + +if (char1 != char2) + { + /* PSHUFD xmm1, xmm2/m128, imm8 */ + instruction[3] = 0xc0 | (cmp2_ind << 3) | cmp2_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + +OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf); + +load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0); +for (i = 0; i < 4; i++) + fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0); + +quit = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + +/* Second part (aligned) */ +start = LABEL(); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); + +add_jump(compiler, ¬_found, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0); +for (i = 0; i < 4; i++) + fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start); + +JUMPHERE(quit); + +/* BSF r32, r/m32 */ +instruction[0] = 0x0f; +instruction[1] = 0xbc; +instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 3); + +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, STR_PTR, 0); +add_jump(compiler, ¬_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0)); + +OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); +return not_found; +} + +#ifndef _WIN64 + +static SLJIT_INLINE sljit_u32 max_fast_forward_char_pair_offset(void) +{ +#if PCRE2_CODE_UNIT_WIDTH == 8 +return 15; +#elif PCRE2_CODE_UNIT_WIDTH == 16 +return 7; +#elif PCRE2_CODE_UNIT_WIDTH == 32 +return 3; +#else +#error "Unsupported unit width" +#endif +} + +#define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) + +static void fast_forward_char_pair_simd(compiler_common *common, sljit_s32 offs1, + PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b) +{ +DEFINE_COMPILER; +sse2_compare_type compare1_type = sse2_compare_match1; +sse2_compare_type compare2_type = sse2_compare_match1; +sljit_u32 bit1 = 0; +sljit_u32 bit2 = 0; +sljit_u32 diff = IN_UCHARS(offs1 - offs2); +sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1); +sljit_s32 tmp2_reg_ind = sljit_get_register_index(TMP2); +sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR); +sljit_s32 data1_ind = 0; +sljit_s32 data2_ind = 1; +sljit_s32 tmp1_ind = 2; +sljit_s32 tmp2_ind = 3; +sljit_s32 cmp1a_ind = 4; +sljit_s32 cmp1b_ind = 5; +sljit_s32 cmp2a_ind = 6; +sljit_s32 cmp2b_ind = 7; +struct sljit_label *start; +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +struct sljit_label *restart; +#endif +struct sljit_jump *jump[2]; +sljit_u8 instruction[8]; +int i; + +SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2); +SLJIT_ASSERT(diff <= IN_UCHARS(max_fast_forward_char_pair_offset())); +SLJIT_ASSERT(tmp1_reg_ind < 8 && tmp2_reg_ind == 1); + +/* Initialize. */ +if (common->match_end_ptr != 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1)); + + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, STR_END, 0); + CMOV(SLJIT_LESS, STR_END, TMP1, 0); + } + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1)); +add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +/* MOVD xmm, r/m32 */ +instruction[0] = 0x66; +instruction[1] = 0x0f; +instruction[2] = 0x6e; + +if (char1a == char1b) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a)); +else + { + bit1 = char1a ^ char1b; + if (is_powerof2(bit1)) + { + compare1_type = sse2_compare_match1i; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a | bit1)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit1)); + } + else + { + compare1_type = sse2_compare_match2; + bit1 = 0; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char1b)); + } + } + +instruction[3] = 0xc0 | (cmp1a_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (char1a != char1b) + { + instruction[3] = 0xc0 | (cmp1b_ind << 3) | tmp2_reg_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +if (char2a == char2b) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a)); +else + { + bit2 = char2a ^ char2b; + if (is_powerof2(bit2)) + { + compare2_type = sse2_compare_match1i; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a | bit2)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit2)); + } + else + { + compare2_type = sse2_compare_match2; + bit2 = 0; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char2b)); + } + } + +instruction[3] = 0xc0 | (cmp2a_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (char2a != char2b) + { + instruction[3] = 0xc0 | (cmp2b_ind << 3) | tmp2_reg_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PSHUFD xmm1, xmm2/m128, imm8 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0x70; +instruction[4] = 0; + +instruction[3] = 0xc0 | (cmp1a_ind << 3) | cmp1a_ind; +sljit_emit_op_custom(compiler, instruction, 5); + +if (char1a != char1b) + { + instruction[3] = 0xc0 | (cmp1b_ind << 3) | cmp1b_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + +instruction[3] = 0xc0 | (cmp2a_ind << 3) | cmp2a_ind; +sljit_emit_op_custom(compiler, instruction, 5); + +if (char2a != char2b) + { + instruction[3] = 0xc0 | (cmp2b_ind << 3) | cmp2b_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +restart = LABEL(); +#endif + +OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, diff); +OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0); +OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); + +load_from_mem_sse2(compiler, data1_ind, str_ptr_reg_ind, 0); + +jump[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_PTR, 0); + +load_from_mem_sse2(compiler, data2_ind, str_ptr_reg_ind, -(sljit_s8)diff); +jump[1] = JUMP(SLJIT_JUMP); + +JUMPHERE(jump[0]); + +/* MOVDQA xmm1, xmm2/m128 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0x6f; +instruction[3] = 0xc0 | (data2_ind << 3) | data1_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +/* PSLLDQ xmm1, imm8 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0x73; +instruction[3] = 0xc0 | (7 << 3) | data2_ind; +instruction[4] = diff; +sljit_emit_op_custom(compiler, instruction, 5); + +JUMPHERE(jump[1]); + +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf); + +for (i = 0; i < 4; i++) + { + fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp2_ind); + fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp1_ind); + } + +/* PAND xmm1, xmm2/m128 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xdb; +instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | 0; +sljit_emit_op_custom(compiler, instruction, 4); + +/* Ignore matches before the first STR_PTR. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0); + +jump[0] = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + +/* Main loop. */ +start = LABEL(); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); +add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +load_from_mem_sse2(compiler, data1_ind, str_ptr_reg_ind, 0); +load_from_mem_sse2(compiler, data2_ind, str_ptr_reg_ind, -(sljit_s8)diff); + +for (i = 0; i < 4; i++) + { + fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp2_ind); + fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp1_ind); + } + +/* PAND xmm1, xmm2/m128 */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xdb; +instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +/* PMOVMSKB reg, xmm */ +/* instruction[0] = 0x66; */ +/* instruction[1] = 0x0f; */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | 0; +sljit_emit_op_custom(compiler, instruction, 4); + +CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start); + +JUMPHERE(jump[0]); + +/* BSF r32, r/m32 */ +instruction[0] = 0x0f; +instruction[1] = 0xbc; +instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind; +sljit_emit_op_custom(compiler, instruction, 3); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + +add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +if (common->match_end_ptr != 0) + OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +if (common->utf) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offs1)); + + jump[0] = jump_if_utf_char_start(compiler, TMP1); + + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, restart); + + add_jump(compiler, &common->failed_match, JUMP(SLJIT_JUMP)); + + JUMPHERE(jump[0]); + } +#endif + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1)); + +if (common->match_end_ptr != 0) + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); +} + +#endif /* !_WIN64 */ + +#undef SSE2_COMPARE_TYPE_INDEX + +#endif /* SLJIT_CONFIG_X86 && !SUPPORT_VALGRIND */ + +#if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64 && (defined __ARM_NEON || defined __ARM_NEON__)) + +#include <arm_neon.h> + +typedef union { + unsigned int x; + struct { unsigned char c1, c2, c3, c4; } c; +} int_char; + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +static SLJIT_INLINE int utf_continue(sljit_u8 *s) +{ +#if PCRE2_CODE_UNIT_WIDTH == 8 +return (*s & 0xc0) == 0x80; +#elif PCRE2_CODE_UNIT_WIDTH == 16 +return (*s & 0xfc00) == 0xdc00; +#else +#error "Unknown code width" +#endif +} +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ + +#if PCRE2_CODE_UNIT_WIDTH == 8 +# define VECTOR_FACTOR 16 +# define vect_t uint8x16_t +# define VLD1Q(X) vld1q_u8((sljit_u8 *)(X)) +# define VCEQQ vceqq_u8 +# define VORRQ vorrq_u8 +# define VST1Q vst1q_u8 +# define VDUPQ vdupq_n_u8 +# define VEXTQ vextq_u8 +# define VANDQ vandq_u8 +typedef union { + uint8_t mem[16]; + uint64_t dw[2]; +} quad_word; +#elif PCRE2_CODE_UNIT_WIDTH == 16 +# define VECTOR_FACTOR 8 +# define vect_t uint16x8_t +# define VLD1Q(X) vld1q_u16((sljit_u16 *)(X)) +# define VCEQQ vceqq_u16 +# define VORRQ vorrq_u16 +# define VST1Q vst1q_u16 +# define VDUPQ vdupq_n_u16 +# define VEXTQ vextq_u16 +# define VANDQ vandq_u16 +typedef union { + uint16_t mem[8]; + uint64_t dw[2]; +} quad_word; +#else +# define VECTOR_FACTOR 4 +# define vect_t uint32x4_t +# define VLD1Q(X) vld1q_u32((sljit_u32 *)(X)) +# define VCEQQ vceqq_u32 +# define VORRQ vorrq_u32 +# define VST1Q vst1q_u32 +# define VDUPQ vdupq_n_u32 +# define VEXTQ vextq_u32 +# define VANDQ vandq_u32 +typedef union { + uint32_t mem[4]; + uint64_t dw[2]; +} quad_word; +#endif + +#define FFCS +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCS + +#define FFCS_2 +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCS_2 + +#define FFCS_MASK +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCS_MASK + +#define JIT_HAS_FAST_FORWARD_CHAR_SIMD 1 + +static void fast_forward_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset) +{ +DEFINE_COMPILER; +int_char ic; +struct sljit_jump *partial_quit; +/* Save temporary registers. */ +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STR_PTR, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP3, 0); + +/* Prepare function arguments */ +OP1(SLJIT_MOV, SLJIT_R0, 0, STR_END, 0); +OP1(SLJIT_MOV, SLJIT_R1, 0, STR_PTR, 0); +OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, offset); + +if (char1 == char2) + { + ic.c.c1 = char1; + ic.c.c2 = char2; + OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x); + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf && offset > 0) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_utf)); + else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs)); +#else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs)); +#endif + } +else + { + PCRE2_UCHAR mask = char1 ^ char2; + if (is_powerof2(mask)) + { + ic.c.c1 = char1 | mask; + ic.c.c2 = mask; + OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x); + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf && offset > 0) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_mask_utf)); + else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_mask)); +#else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_mask)); +#endif + } + else + { + ic.c.c1 = char1; + ic.c.c2 = char2; + OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x); + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf && offset > 0) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_2_utf)); + else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_2)); +#else + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(UW) | SLJIT_ARG3(UW) | SLJIT_ARG4(UW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcs_2)); +#endif + } + } +/* Restore registers. */ +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); +OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); + +/* Check return value. */ +partial_quit = CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +if (common->mode == PCRE2_JIT_COMPLETE) + add_jump(compiler, &common->failed_match, partial_quit); + +/* Fast forward STR_PTR to the result of memchr. */ +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); + +if (common->mode != PCRE2_JIT_COMPLETE) + JUMPHERE(partial_quit); +} + +typedef enum { + compare_match1, + compare_match1i, + compare_match2, +} compare_type; + +static inline vect_t fast_forward_char_pair_compare(compare_type ctype, vect_t dst, vect_t cmp1, vect_t cmp2) +{ +if (ctype == compare_match2) + { + vect_t tmp = dst; + dst = VCEQQ(dst, cmp1); + tmp = VCEQQ(tmp, cmp2); + dst = VORRQ(dst, tmp); + return dst; + } + +if (ctype == compare_match1i) + dst = VORRQ(dst, cmp2); +dst = VCEQQ(dst, cmp1); +return dst; +} + +static SLJIT_INLINE sljit_u32 max_fast_forward_char_pair_offset(void) +{ +#if PCRE2_CODE_UNIT_WIDTH == 8 +return 15; +#elif PCRE2_CODE_UNIT_WIDTH == 16 +return 7; +#elif PCRE2_CODE_UNIT_WIDTH == 32 +return 3; +#else +#error "Unsupported unit width" +#endif +} + +/* ARM doesn't have a shift left across lanes. */ +static SLJIT_INLINE vect_t shift_left_n_lanes(vect_t a, sljit_u8 n) +{ +vect_t zero = VDUPQ(0); +SLJIT_ASSERT(0 < n && n < VECTOR_FACTOR); +/* VEXTQ takes an immediate as last argument. */ +#define C(X) case X: return VEXTQ(zero, a, VECTOR_FACTOR - X); +switch (n) + { + C(1); C(2); C(3); +#if PCRE2_CODE_UNIT_WIDTH != 32 + C(4); C(5); C(6); C(7); +# if PCRE2_CODE_UNIT_WIDTH != 16 + C(8); C(9); C(10); C(11); C(12); C(13); C(14); C(15); +# endif +#endif + default: + /* Based on the ASSERT(0 < n && n < VECTOR_FACTOR) above, this won't + happen. The return is still here for compilers to not warn. */ + return a; + } +} + +#define FFCPS +#define FFCPS_DIFF1 +#define FFCPS_CHAR1A2A + +#define FFCPS_0 +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCPS_0 + +#undef FFCPS_CHAR1A2A + +#define FFCPS_1 +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCPS_1 + +#undef FFCPS_DIFF1 + +#define FFCPS_DEFAULT +#include "pcre2_jit_neon_inc.h" +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +# define FF_UTF +# include "pcre2_jit_neon_inc.h" +# undef FF_UTF +#endif +#undef FFCPS + +#define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD 1 + +static void fast_forward_char_pair_simd(compiler_common *common, sljit_s32 offs1, + PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b) +{ +DEFINE_COMPILER; +sljit_u32 diff = IN_UCHARS(offs1 - offs2); +struct sljit_jump *partial_quit; +int_char ic; +SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2); +SLJIT_ASSERT(diff <= IN_UCHARS(max_fast_forward_char_pair_offset())); +SLJIT_ASSERT(compiler->scratches == 5); + +/* Save temporary register STR_PTR. */ +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STR_PTR, 0); + +/* Prepare arguments for the function call. */ +if (common->match_end_ptr == 0) + OP1(SLJIT_MOV, SLJIT_R0, 0, STR_END, 0); +else + { + OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + OP2(SLJIT_ADD, SLJIT_R0, 0, SLJIT_R0, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1)); + + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, STR_END, 0, SLJIT_R0, 0); + CMOV(SLJIT_LESS, SLJIT_R0, STR_END, 0); + } + +OP1(SLJIT_MOV, SLJIT_R1, 0, STR_PTR, 0); +OP1(SLJIT_MOV_S32, SLJIT_R2, 0, SLJIT_IMM, offs1); +OP1(SLJIT_MOV_S32, SLJIT_R3, 0, SLJIT_IMM, offs2); +ic.c.c1 = char1a; +ic.c.c2 = char1b; +ic.c.c3 = char2a; +ic.c.c4 = char2b; +OP1(SLJIT_MOV_U32, SLJIT_R4, 0, SLJIT_IMM, ic.x); + +if (diff == 1) { + if (char1a == char1b && char2a == char2b) { +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_0_utf)); + else +#endif + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_0)); + } else { +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_1_utf)); + else +#endif + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_1)); + } +} else { +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 + if (common->utf) + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_default_utf)); + else +#endif + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), + SLJIT_IMM, SLJIT_FUNC_OFFSET(ffcps_default)); +} + +/* Restore STR_PTR register. */ +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + +/* Check return value. */ +partial_quit = CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +add_jump(compiler, &common->failed_match, partial_quit); + +/* Fast forward STR_PTR to the result of memchr. */ +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); + +JUMPHERE(partial_quit); +} + +#endif /* SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64 */ diff --git a/dist2/src/pcre2_jit_test.c b/dist2/src/pcre2_jit_test.c index fa14329a..d9358873 100644 --- a/dist2/src/pcre2_jit_test.c +++ b/dist2/src/pcre2_jit_test.c @@ -196,6 +196,8 @@ static struct regression_test_case regression_test_cases[] = { { CMUP, A, 0, 0, "\xf0\x90\x90\xa8{2}", "\xf0\x90\x90\x80#\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, { CMUP, A, 0, 0, "\xe1\xbd\xb8\xe1\xbf\xb8", "\xe1\xbf\xb8\xe1\xbd\xb8" }, { M, A, 0, 0, "[3-57-9]", "5" }, + { PCRE2_AUTO_CALLOUT, A, 0, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", + "12345678901234567890123456789012345678901234567890123456789012345678901234567890" }, /* Assertions. */ { MU, A, 0, 0, "\\b[^A]", "A_B#" }, @@ -348,6 +350,7 @@ static struct regression_test_case regression_test_cases[] = { { MU, A, 0, 0, ".[ab]*.", "xx" }, { MU, A, 0, 0, ".[ab]*a", "xxa" }, { MU, A, 0, 0, ".[ab]?.", "xx" }, + { MU, A, 0, 0, "_[ab]+_*a", "_aa" }, /* Bracket repeats with limit. */ { MU, A, 0, 0, "(?:(ab){2}){5}M", "abababababababababababM" }, @@ -406,6 +409,7 @@ static struct regression_test_case regression_test_cases[] = { { MUP, A, 0, 0 | F_PROPERTY, "[\xc3\xa2-\xc3\xa6\xc3\x81-\xc3\x84\xe2\x80\xa8-\xe2\x80\xa9\xe6\x92\xad\\p{Zs}]{2,}", "\xe2\x80\xa7\xe2\x80\xa9\xe6\x92\xad \xe6\x92\xae" }, { MUP, A, 0, 0 | F_PROPERTY, "[\\P{L&}]{2}[^\xc2\x85-\xc2\x89\\p{Ll}\\p{Lu}]{2}", "\xc3\xa9\xe6\x92\xad.a\xe6\x92\xad|\xc2\x8a#" }, { PCRE2_UCP, 0, 0, 0 | F_PROPERTY, "[a-b\\s]{2,5}[^a]", "AB baaa" }, + { MUP, 0, 0, 0 | F_NOMATCH, "[^\\p{Hangul}\\p{Z}]", " " }, /* Possible empty brackets. */ { MU, A, 0, 0, "(?:|ab||bc|a)+d", "abcxabcabd" }, @@ -636,6 +640,7 @@ static struct regression_test_case regression_test_cases[] = { { MU, A, 0, 0, "(?=(?:x|ab(*ACCEPT)b))", "ab" }, { MU, A, 0, 0, "(?=(a(b(*ACCEPT)b)))a", "ab" }, { MU, A, PCRE2_NOTEMPTY, 0, "(?=a*(*ACCEPT))c", "c" }, + { MU, A, PCRE2_NOTEMPTY, 0 | F_NOMATCH, "(?=A)", "AB" }, /* Conditional blocks. */ { MU, A, 0, 0, "(?(?=(a))a|b)+k", "ababbalbbadabak" }, @@ -857,6 +862,8 @@ static struct regression_test_case regression_test_cases[] = { { MU, A, 0, 0, "(?(?!a(*THEN)b)ad|add)", "add" }, { MU, A, 0, 0 | F_NOMATCH, "(?(?=a)a(*THEN)b|ad)", "ad" }, { MU, A, 0, 0, "(?!(?(?=a)ab|b(*THEN)d))bn|bnn", "bnn" }, + { MU, A, 0, 0, "(?=(*THEN: ))* ", " " }, + { MU, A, 0, 0, "a(*THEN)(?R) |", "a" }, /* Recurse and control verbs. */ { MU, A, 0, 0, "(a(*ACCEPT)b){0}a(?1)b", "aacaabb" }, @@ -1351,10 +1358,11 @@ static int regression_tests(void) ovector8_1[i] = -2; for (i = 0; i < OVECTOR_SIZE * 2; ++i) ovector8_2[i] = -2; + pcre2_set_match_limit_8(mcontext8, 10000000); } if (re8) { return_value8[1] = pcre2_match_8(re8, (PCRE2_SPTR8)current->input, strlen(current->input), - current->start_offset & OFFSET_MASK, current->match_options, mdata8_2, NULL); + current->start_offset & OFFSET_MASK, current->match_options, mdata8_2, mcontext8); if (pcre2_jit_compile_8(re8, jit_compile_mode)) { printf("\n8 bit: JIT compiler does not support \"%s\"\n", current->pattern); @@ -1390,6 +1398,7 @@ static int regression_tests(void) ovector16_1[i] = -2; for (i = 0; i < OVECTOR_SIZE * 2; ++i) ovector16_2[i] = -2; + pcre2_set_match_limit_16(mcontext16, 10000000); } if (re16) { if ((current->compile_options & PCRE2_UTF) || (current->start_offset & F_FORCECONV)) @@ -1398,7 +1407,7 @@ static int regression_tests(void) length16 = copy_char8_to_char16((PCRE2_SPTR8)current->input, regtest_buf16, REGTEST_MAX_LENGTH16); return_value16[1] = pcre2_match_16(re16, regtest_buf16, length16, - current->start_offset & OFFSET_MASK, current->match_options, mdata16_2, NULL); + current->start_offset & OFFSET_MASK, current->match_options, mdata16_2, mcontext16); if (pcre2_jit_compile_16(re16, jit_compile_mode)) { printf("\n16 bit: JIT compiler does not support \"%s\"\n", current->pattern); @@ -1434,6 +1443,7 @@ static int regression_tests(void) ovector32_1[i] = -2; for (i = 0; i < OVECTOR_SIZE * 2; ++i) ovector32_2[i] = -2; + pcre2_set_match_limit_32(mcontext32, 10000000); } if (re32) { if ((current->compile_options & PCRE2_UTF) || (current->start_offset & F_FORCECONV)) @@ -1442,7 +1452,7 @@ static int regression_tests(void) length32 = copy_char8_to_char32((PCRE2_SPTR8)current->input, regtest_buf32, REGTEST_MAX_LENGTH32); return_value32[1] = pcre2_match_32(re32, regtest_buf32, length32, - current->start_offset & OFFSET_MASK, current->match_options, mdata32_2, NULL); + current->start_offset & OFFSET_MASK, current->match_options, mdata32_2, mcontext32); if (pcre2_jit_compile_32(re32, jit_compile_mode)) { printf("\n32 bit: JIT compiler does not support \"%s\"\n", current->pattern); @@ -1770,7 +1780,7 @@ static int regression_tests(void) } } -#if defined SUPPORT_UNICODE && (defined SUPPORT_PCRE2_8 || defined SUPPORT_PCRE2_16 || defined SUPPORT_PCRE2_32) +#if defined SUPPORT_UNICODE static int check_invalid_utf_result(int pattern_index, const char *type, int result, int match_start, int match_end, PCRE2_SIZE *ovector) @@ -1803,7 +1813,7 @@ static int check_invalid_utf_result(int pattern_index, const char *type, int res return 0; } -#endif /* SUPPORT_UNICODE && (SUPPORT_PCRE2_8 || SUPPORT_PCRE2_16 || SUPPORT_PCRE2_32) */ +#endif /* SUPPORT_UNICODE */ #if defined SUPPORT_UNICODE && defined SUPPORT_PCRE2_8 @@ -1823,7 +1833,9 @@ struct invalid_utf8_regression_test_case { const char *input; }; -static struct invalid_utf8_regression_test_case invalid_utf8_regression_test_cases[] = { +static const char invalid_utf8_newline_cr; + +static const struct invalid_utf8_regression_test_case invalid_utf8_regression_test_cases[] = { { UDA, CI, 0, 0, 0, 0, 4, { ".", NULL }, "\xf4\x8f\xbf\xbf" }, { UDA, CI, 0, 0, 0, 0, 4, { ".", NULL }, "\xf0\x90\x80\x80" }, { UDA, CI, 0, 0, 0, -1, -1, { ".", NULL }, "\xf4\x90\x80\x80" }, @@ -1864,53 +1876,52 @@ static struct invalid_utf8_regression_test_case invalid_utf8_regression_test_cas { UDA, CI, 0, 0, 0, 0, 1, { ".", NULL }, "\x7f" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "\xf4\x8f\xbf\xbf#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\xa0\x80\x80#" }, - { UDA, CPI, 4, 1, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xbf\xbf#" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\xa0\x80\x80\xf4\xa0\x80\x80" }, + { UDA, CPI, 4, 1, 1, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbf" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "#\xef\xbf\xbf#" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "#\xe0\xa0\x80#" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "\xf0\x90\x80\x80#" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "\xf3\xbf\xbf\xbf#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf0\x8f\xbf\xbf#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf5\x80\x80\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x90\x80\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xbf\xff#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xff\xbf#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xef\x80\x80\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\x80\x80\x80\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xe0\x9f\xbf#" }, - { UDA, CPI, 4, 2, 0, -1, -1, { "\\B", "\\b" }, "#\xe0\xa0\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xf0\x80\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xed\xa0\x80#" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf0\x8f\xbf\xbf\xf0\x8f\xbf\xbf" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf5\x80\x80\x80\xf5\x80\x80\x80" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x90\x80\x80\xf4\x90\x80\x80" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xbf\xff\xf4\x8f\xbf\xff" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\xf4\x8f\xff\xbf\xf4\x8f\xff\xbf" }, + { UDA, CPI, 4, 0, 1, -1, -1, { "\\B", "\\b" }, "\xef\x80\x80\x80\xef\x80\x80" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "\x80\x80\x80\x80\x80\x80\x80\x80" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xe0\x9f\xbf\xe0\x9f\xbf#" }, + { UDA, CPI, 4, 2, 2, -1, -1, { "\\B", "\\b" }, "#\xe0\xa0\x80\xe0\xa0\x80#" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xf0\x80\x80\xf0\x80\x80#" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "#\xed\xa0\x80\xed\xa0\x80#" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "##\xdf\xbf#" }, { UDA, CPI, 4, 2, 0, 2, 2, { "\\B", NULL }, "##\xdf\xbf#" }, { UDA, CPI, 4, 0, 0, 4, 4, { "\\B", NULL }, "##\xc2\x80#" }, { UDA, CPI, 4, 2, 0, 2, 2, { "\\B", NULL }, "##\xc2\x80#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xc1\xbf#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xdf\xc0#" }, - { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xe0\x80#" }, - { UDA, CPI, 4, 2, 0, -1, -1, { "\\B", "\\b" }, "##\xe0\x80#" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xc1\xbf\xc1\xbf##" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xdf\xc0\xdf\xc0##" }, + { UDA, CPI, 4, 0, 0, -1, -1, { "\\B", "\\b" }, "##\xe0\x80\xe0\x80##" }, { UDA, CPI, 3, 0, 0, 3, 3, { "\\B", NULL }, "\xef\xbf\xbf#" }, { UDA, CPI, 3, 0, 0, 3, 3, { "\\B", NULL }, "\xe0\xa0\x80#" }, - { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xe0\x9f\xbf#" }, - { UDA, CPI, 3, 1, 0, -1, -1, { "\\B", "\\b" }, "\xef\xbf\xbf#" }, - { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xdf\x80\x80#" }, - { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xef\xbf\xff#" }, - { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xef\xff\xbf#" }, - { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xed\xbf\xbf#" }, + { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xe0\x9f\xbf\xe0\x9f\xbf" }, + { UDA, CPI, 3, 1, 1, -1, -1, { "\\B", "\\b" }, "\xef\xbf\xbf\xef\xbf\xbf" }, + { UDA, CPI, 3, 0, 1, -1, -1, { "\\B", "\\b" }, "\xdf\x80\x80\xdf\x80" }, + { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xef\xbf\xff\xef\xbf\xff" }, + { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xef\xff\xbf\xef\xff\xbf" }, + { UDA, CPI, 3, 0, 0, -1, -1, { "\\B", "\\b" }, "\xed\xbf\xbf\xed\xbf\xbf" }, { UDA, CPI, 2, 0, 0, 2, 2, { "\\B", NULL }, "\xdf\xbf#" }, { UDA, CPI, 2, 0, 0, 2, 2, { "\\B", NULL }, "\xc2\x80#" }, - { UDA, CPI, 2, 1, 0, -1, -1, { "\\B", "\\b" }, "\xdf\xbf#" }, - { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xc1\xbf#" }, - { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xe0\x80#" }, - { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xdf\xff#" }, - { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xff\xbf#" }, + { UDA, CPI, 2, 1, 1, -1, -1, { "\\B", "\\b" }, "\xdf\xbf\xdf\xbf" }, + { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xc1\xbf\xc1\xbf" }, + { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xe0\x80\xe0\x80" }, + { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xdf\xff\xdf\xff" }, + { UDA, CPI, 2, 0, 0, -1, -1, { "\\B", "\\b" }, "\xff\xbf\xff\xbf" }, { UDA, CPI, 1, 0, 0, 1, 1, { "\\B", NULL }, "\x7f#" }, { UDA, CPI, 1, 0, 0, 1, 1, { "\\B", NULL }, "\x01#" }, - { UDA, CPI, 1, 0, 0, -1, -1, { "\\B", "\\b" }, "\x80#" }, - { UDA, CPI, 1, 0, 0, -1, -1, { "\\B", "\\b" }, "\x80#" }, + { UDA, CPI, 1, 0, 0, -1, -1, { "\\B", "\\b" }, "\x80\x80" }, + { UDA, CPI, 1, 0, 0, -1, -1, { "\\B", "\\b" }, "\xb0\xb0" }, { UDA | PCRE2_CASELESS, CPI, 0, 0, 0, 0, 2, { "(.)\\1", NULL }, "aA" }, { UDA | PCRE2_CASELESS, CPI, 0, 0, 0, -1, -1, { "(.)\\1", NULL }, "a\xff" }, @@ -1961,10 +1972,14 @@ static struct invalid_utf8_regression_test_case invalid_utf8_regression_test_cas { PCRE2_UTF, CI, 0, 0, 0, 4, 8, { "#\xc7\x85#", NULL }, "\x80\x80#\xc7#\xc7\x85#" }, { PCRE2_UTF, CI, 0, 0, 0, 7, 11, { "#\xc7\x85#", NULL }, "\x80\x80#\xc7\x80\x80\x80#\xc7\x85#" }, + { PCRE2_UTF | PCRE2_UCP, CI, 0, 0, 0, -1, -1, { "[\\s]", NULL }, "\xed\xa0\x80" }, + /* These two are not invalid UTF tests, but this infrastructure fits better for them. */ { 0, PCRE2_JIT_COMPLETE, 0, 0, 1, -1, -1, { "\\X{2}", NULL }, "\r\n\n" }, { 0, PCRE2_JIT_COMPLETE, 0, 0, 1, -1, -1, { "\\R{2}", NULL }, "\r\n\n" }, + { PCRE2_UTF | PCRE2_MULTILINE, CI, 0, 0, 0, -1, -1, { "^.a", &invalid_utf8_newline_cr }, "\xc3\xa7#a" }, + { 0, 0, 0, 0, 0, 0, 0, { NULL, NULL }, NULL } }; @@ -1972,7 +1987,7 @@ static struct invalid_utf8_regression_test_case invalid_utf8_regression_test_cas #undef CI #undef CPI -static int run_invalid_utf8_test(struct invalid_utf8_regression_test_case *current, +static int run_invalid_utf8_test(const struct invalid_utf8_regression_test_case *current, int pattern_index, int i, pcre2_compile_context_8 *ccontext, pcre2_match_data_8 *mdata) { pcre2_code_8 *code; @@ -2025,7 +2040,7 @@ static int run_invalid_utf8_test(struct invalid_utf8_regression_test_case *curre static int invalid_utf8_regression_tests(void) { - struct invalid_utf8_regression_test_case *current; + const struct invalid_utf8_regression_test_case *current; pcre2_compile_context_8 *ccontext; pcre2_match_data_8 *mdata; int total = 0, successful = 0; @@ -2042,10 +2057,18 @@ static int invalid_utf8_regression_tests(void) total++; result = 1; - if (!run_invalid_utf8_test(current, total - 1, 0, ccontext, mdata)) - result = 0; - if (!run_invalid_utf8_test(current, total - 1, 1, ccontext, mdata)) - result = 0; + if (current->pattern[1] != &invalid_utf8_newline_cr) + { + if (!run_invalid_utf8_test(current, total - 1, 0, ccontext, mdata)) + result = 0; + if (!run_invalid_utf8_test(current, total - 1, 1, ccontext, mdata)) + result = 0; + } else { + pcre2_set_newline_8(ccontext, PCRE2_NEWLINE_CR); + if (!run_invalid_utf8_test(current, total - 1, 0, ccontext, mdata)) + result = 0; + pcre2_set_newline_8(ccontext, PCRE2_NEWLINE_ANY); + } if (result) { successful++; @@ -2107,10 +2130,10 @@ static PCRE2_UCHAR16 nothashmark16[] = { '[', '^', '#', ']', 0 }; static PCRE2_UCHAR16 afternl16[] = { '^', '\\', 'W', 0 }; static PCRE2_UCHAR16 generic16[] = { '#', 0xd800, 0xdc00, '#', 0 }; static PCRE2_UCHAR16 test16_1[] = { 0xd7ff, 0xe000, 0xffff, 0x01, '#', 0 }; -static PCRE2_UCHAR16 test16_2[] = { 0xd800, 0xdc00, '#', 0 }; -static PCRE2_UCHAR16 test16_3[] = { 0xdbff, 0xdfff, '#', 0 }; -static PCRE2_UCHAR16 test16_4[] = { 0xd800, 0xdbff, '#', 0 }; -static PCRE2_UCHAR16 test16_5[] = { '#', 0xd800, '#', 0 }; +static PCRE2_UCHAR16 test16_2[] = { 0xd800, 0xdc00, 0xd800, 0xdc00, 0 }; +static PCRE2_UCHAR16 test16_3[] = { 0xdbff, 0xdfff, 0xdbff, 0xdfff, 0 }; +static PCRE2_UCHAR16 test16_4[] = { 0xd800, 0xdbff, 0xd800, 0xdbff, 0 }; +static PCRE2_UCHAR16 test16_5[] = { '#', 0xd800, 0xdc00, '#', 0 }; static PCRE2_UCHAR16 test16_6[] = { 'a', 'A', 0xdc28, 0 }; static PCRE2_UCHAR16 test16_7[] = { 0xd801, 0xdc00, 0xd801, 0xdc28, 0 }; static PCRE2_UCHAR16 test16_8[] = { '#', 0xd800, 0xdc00, 0 }; @@ -2119,16 +2142,16 @@ static PCRE2_UCHAR16 test16_10[] = { ' ', 0xdc00, 0xd800, 0x2028, '#', 0 }; static PCRE2_UCHAR16 test16_11[] = { 0xdc00, 0xdc00, 0xd800, 0xdc00, 0xdc00, '#', 0xd800, 0xdc00, '#', 0 }; static PCRE2_UCHAR16 test16_12[] = { '#', 0xd800, 0xdc00, 0xd800, '#', 0xd800, 0xdc00, 0xdc00, 0xdc00, '#', 0xd800, 0xdc00, '#', 0 }; -static struct invalid_utf16_regression_test_case invalid_utf16_regression_test_cases[] = { +static const struct invalid_utf16_regression_test_case invalid_utf16_regression_test_cases[] = { { UDA, CI, 0, 0, 0, 0, 1, { allany16, NULL }, test16_1 }, { UDA, CI, 1, 0, 0, 1, 2, { allany16, NULL }, test16_1 }, { UDA, CI, 2, 0, 0, 2, 3, { allany16, NULL }, test16_1 }, { UDA, CI, 3, 0, 0, 3, 4, { allany16, NULL }, test16_1 }, { UDA, CI, 0, 0, 0, 0, 2, { allany16, NULL }, test16_2 }, - { UDA, CI, 0, 0, 2, -1, -1, { allany16, NULL }, test16_2 }, + { UDA, CI, 0, 0, 3, -1, -1, { allany16, NULL }, test16_2 }, { UDA, CI, 1, 0, 0, -1, -1, { allany16, NULL }, test16_2 }, { UDA, CI, 0, 0, 0, 0, 2, { allany16, NULL }, test16_3 }, - { UDA, CI, 0, 0, 2, -1, -1, { allany16, NULL }, test16_3 }, + { UDA, CI, 0, 0, 3, -1, -1, { allany16, NULL }, test16_3 }, { UDA, CI, 1, 0, 0, -1, -1, { allany16, NULL }, test16_3 }, { UDA, CPI, 1, 0, 0, 1, 1, { non_word_boundary16, NULL }, test16_1 }, @@ -2137,8 +2160,8 @@ static struct invalid_utf16_regression_test_case invalid_utf16_regression_test_c { UDA, CPI, 4, 0, 0, 4, 4, { non_word_boundary16, NULL }, test16_1 }, { UDA, CPI, 2, 0, 0, 2, 2, { non_word_boundary16, NULL }, test16_2 }, { UDA, CPI, 2, 0, 0, 2, 2, { non_word_boundary16, NULL }, test16_3 }, - { UDA, CPI, 2, 1, 0, -1, -1, { non_word_boundary16, word_boundary16 }, test16_2 }, - { UDA, CPI, 2, 1, 0, -1, -1, { non_word_boundary16, word_boundary16 }, test16_3 }, + { UDA, CPI, 2, 1, 1, -1, -1, { non_word_boundary16, word_boundary16 }, test16_2 }, + { UDA, CPI, 2, 1, 1, -1, -1, { non_word_boundary16, word_boundary16 }, test16_3 }, { UDA, CPI, 2, 0, 0, -1, -1, { non_word_boundary16, word_boundary16 }, test16_4 }, { UDA, CPI, 2, 0, 0, -1, -1, { non_word_boundary16, word_boundary16 }, test16_5 }, @@ -2173,7 +2196,7 @@ static struct invalid_utf16_regression_test_case invalid_utf16_regression_test_c #undef CI #undef CPI -static int run_invalid_utf16_test(struct invalid_utf16_regression_test_case *current, +static int run_invalid_utf16_test(const struct invalid_utf16_regression_test_case *current, int pattern_index, int i, pcre2_compile_context_16 *ccontext, pcre2_match_data_16 *mdata) { pcre2_code_16 *code; @@ -2233,7 +2256,7 @@ static int run_invalid_utf16_test(struct invalid_utf16_regression_test_case *cur static int invalid_utf16_regression_tests(void) { - struct invalid_utf16_regression_test_case *current; + const struct invalid_utf16_regression_test_case *current; pcre2_compile_context_16 *ccontext; pcre2_match_data_16 *mdata; int total = 0, successful = 0; @@ -2313,32 +2336,45 @@ static PCRE2_UCHAR32 backreference32[] = { '(', '.', ')', '\\', '1', 0 }; static PCRE2_UCHAR32 grapheme32[] = { '\\', 'X', 0 }; static PCRE2_UCHAR32 nothashmark32[] = { '[', '^', '#', ']', 0 }; static PCRE2_UCHAR32 afternl32[] = { '^', '\\', 'W', 0 }; -static PCRE2_UCHAR32 test32_1[] = { 0x10ffff, 0x10ffff, 0x110000, 0x10ffff, 0 }; -static PCRE2_UCHAR32 test32_2[] = { 'a', 'A', 0x110000, 0 }; -static PCRE2_UCHAR32 test32_3[] = { '#', 0x10ffff, 0x110000, 0 }; -static PCRE2_UCHAR32 test32_4[] = { ' ', 0x2028, '#', 0 }; -static PCRE2_UCHAR32 test32_5[] = { ' ', 0x110000, 0x2028, '#', 0 }; - -static struct invalid_utf32_regression_test_case invalid_utf32_regression_test_cases[] = { +static PCRE2_UCHAR32 test32_1[] = { 0x10ffff, 0x10ffff, 0x110000, 0x110000, 0x10ffff, 0 }; +static PCRE2_UCHAR32 test32_2[] = { 0xd7ff, 0xe000, 0xd800, 0xdfff, 0xe000, 0xdfff, 0xd800, 0 }; +static PCRE2_UCHAR32 test32_3[] = { 'a', 'A', 0x110000, 0 }; +static PCRE2_UCHAR32 test32_4[] = { '#', 0x10ffff, 0x110000, 0 }; +static PCRE2_UCHAR32 test32_5[] = { ' ', 0x2028, '#', 0 }; +static PCRE2_UCHAR32 test32_6[] = { ' ', 0x110000, 0x2028, '#', 0 }; + +static const struct invalid_utf32_regression_test_case invalid_utf32_regression_test_cases[] = { { UDA, CI, 0, 0, 0, 0, 1, { allany32, NULL }, test32_1 }, { UDA, CI, 2, 0, 0, -1, -1, { allany32, NULL }, test32_1 }, + { UDA, CI, 0, 0, 0, 0, 1, { allany32, NULL }, test32_2 }, + { UDA, CI, 1, 0, 0, 1, 2, { allany32, NULL }, test32_2 }, + { UDA, CI, 2, 0, 0, -1, -1, { allany32, NULL }, test32_2 }, + { UDA, CI, 3, 0, 0, -1, -1, { allany32, NULL }, test32_2 }, { UDA, CPI, 1, 0, 0, 1, 1, { non_word_boundary32, NULL }, test32_1 }, - { UDA, CPI, 2, 0, 0, -1, -1, { non_word_boundary32, word_boundary32 }, test32_1 }, { UDA, CPI, 3, 0, 0, -1, -1, { non_word_boundary32, word_boundary32 }, test32_1 }, + { UDA, CPI, 1, 0, 0, 1, 1, { non_word_boundary32, NULL }, test32_2 }, + { UDA, CPI, 3, 0, 0, -1, -1, { non_word_boundary32, word_boundary32 }, test32_2 }, + { UDA, CPI, 6, 0, 0, -1, -1, { non_word_boundary32, word_boundary32 }, test32_2 }, - { UDA | PCRE2_CASELESS, CPI, 0, 0, 0, 0, 2, { backreference32, NULL }, test32_2 }, - { UDA | PCRE2_CASELESS, CPI, 1, 0, 0, -1, -1, { backreference32, NULL }, test32_2 }, + { UDA | PCRE2_CASELESS, CPI, 0, 0, 0, 0, 2, { backreference32, NULL }, test32_3 }, + { UDA | PCRE2_CASELESS, CPI, 1, 0, 0, -1, -1, { backreference32, NULL }, test32_3 }, { UDA, CPI, 0, 0, 0, 0, 1, { grapheme32, NULL }, test32_1 }, { UDA, CPI, 2, 0, 0, -1, -1, { grapheme32, NULL }, test32_1 }, + { UDA, CPI, 1, 0, 0, 1, 2, { grapheme32, NULL }, test32_2 }, + { UDA, CPI, 2, 0, 0, -1, -1, { grapheme32, NULL }, test32_2 }, + { UDA, CPI, 3, 0, 0, -1, -1, { grapheme32, NULL }, test32_2 }, + { UDA, CPI, 4, 0, 0, 4, 5, { grapheme32, NULL }, test32_2 }, - { UDA, CPI, 0, 0, 0, -1, -1, { nothashmark32, NULL }, test32_3 }, - { UDA, CPI, 1, 0, 0, 1, 2, { nothashmark32, NULL }, test32_3 }, - { UDA, CPI, 2, 0, 0, -1, -1, { nothashmark32, NULL }, test32_3 }, + { UDA, CPI, 0, 0, 0, -1, -1, { nothashmark32, NULL }, test32_4 }, + { UDA, CPI, 1, 0, 0, 1, 2, { nothashmark32, NULL }, test32_4 }, + { UDA, CPI, 2, 0, 0, -1, -1, { nothashmark32, NULL }, test32_4 }, + { UDA, CPI, 1, 0, 0, 1, 2, { nothashmark32, NULL }, test32_2 }, + { UDA, CPI, 2, 0, 0, -1, -1, { nothashmark32, NULL }, test32_2 }, - { PCRE2_UTF | PCRE2_MULTILINE, CI, 1, 0, 0, 2, 3, { afternl32, NULL }, test32_4 }, - { PCRE2_UTF | PCRE2_MULTILINE, CI, 1, 0, 0, 3, 4, { afternl32, NULL }, test32_5 }, + { PCRE2_UTF | PCRE2_MULTILINE, CI, 1, 0, 0, 2, 3, { afternl32, NULL }, test32_5 }, + { PCRE2_UTF | PCRE2_MULTILINE, CI, 1, 0, 0, 3, 4, { afternl32, NULL }, test32_6 }, { 0, 0, 0, 0, 0, 0, 0, { NULL, NULL }, NULL } }; @@ -2347,7 +2383,7 @@ static struct invalid_utf32_regression_test_case invalid_utf32_regression_test_c #undef CI #undef CPI -static int run_invalid_utf32_test(struct invalid_utf32_regression_test_case *current, +static int run_invalid_utf32_test(const struct invalid_utf32_regression_test_case *current, int pattern_index, int i, pcre2_compile_context_32 *ccontext, pcre2_match_data_32 *mdata) { pcre2_code_32 *code; @@ -2407,7 +2443,7 @@ static int run_invalid_utf32_test(struct invalid_utf32_regression_test_case *cur static int invalid_utf32_regression_tests(void) { - struct invalid_utf32_regression_test_case *current; + const struct invalid_utf32_regression_test_case *current; pcre2_compile_context_32 *ccontext; pcre2_match_data_32 *mdata; int total = 0, successful = 0; diff --git a/dist2/src/pcre2_maketables.c b/dist2/src/pcre2_maketables.c index 5921e907..56d24940 100644 --- a/dist2/src/pcre2_maketables.c +++ b/dist2/src/pcre2_maketables.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -41,10 +41,11 @@ POSSIBILITY OF SUCH DAMAGE. /* This module contains the external function pcre2_maketables(), which builds character tables for PCRE2 in the current locale. The file is compiled on its -own as part of the PCRE2 library. However, it is also included in the -compilation of dftables.c, in which case the macro DFTABLES is defined. */ +own as part of the PCRE2 library. It is also included in the compilation of +pcre2_dftables.c as a freestanding program, in which case the macro +PCRE2_DFTABLES is defined. */ -#ifndef DFTABLES +#ifndef PCRE2_DFTABLES /* Compiling the library */ # ifdef HAVE_CONFIG_H # include "config.h" # endif @@ -61,28 +62,29 @@ compilation of dftables.c, in which case the macro DFTABLES is defined. */ a pointer to them. They are build using the ctype functions, and consequently their contents will depend upon the current locale setting. When compiled as part of the library, the store is obtained via a general context malloc, if -supplied, but when DFTABLES is defined (when compiling the dftables auxiliary -program) malloc() is used, and the function has a different name so as not to -clash with the prototype in pcre2.h. +supplied, but when PCRE2_DFTABLES is defined (when compiling the pcre2_dftables +freestanding auxiliary program) malloc() is used, and the function has a +different name so as not to clash with the prototype in pcre2.h. -Arguments: none when DFTABLES is defined - else a PCRE2 general context or NULL +Arguments: none when PCRE2_DFTABLES is defined + else a PCRE2 general context or NULL Returns: pointer to the contiguous block of data + else NULL if memory allocation failed */ -#ifdef DFTABLES /* Included in freestanding dftables.c program */ +#ifdef PCRE2_DFTABLES /* Included in freestanding pcre2_dftables program */ static const uint8_t *maketables(void) { -uint8_t *yield = (uint8_t *)malloc(tables_length); +uint8_t *yield = (uint8_t *)malloc(TABLES_LENGTH); -#else /* Not DFTABLES, compiling the library */ +#else /* Not PCRE2_DFTABLES, that is, compiling the library */ PCRE2_EXP_DEFN const uint8_t * PCRE2_CALL_CONVENTION pcre2_maketables(pcre2_general_context *gcontext) { uint8_t *yield = (uint8_t *)((gcontext != NULL)? - gcontext->memctl.malloc(tables_length, gcontext->memctl.memory_data) : - malloc(tables_length)); -#endif /* DFTABLES */ + gcontext->memctl.malloc(TABLES_LENGTH, gcontext->memctl.memory_data) : + malloc(TABLES_LENGTH)); +#endif /* PCRE2_DFTABLES */ int i; uint8_t *p; @@ -103,8 +105,8 @@ exclusive ones - in some locales things may be different. Note that the table for "space" includes everything "isspace" gives, including VT in the default locale. This makes it work for the POSIX class [:space:]. -From release 8.34 is is also correct for Perl space, because Perl added VT at -release 5.18. +From PCRE1 release 8.34 and for all PCRE2 releases it is also correct for Perl +space, because Perl added VT at release 5.18. Note also that it is possible for a character to be alnum or alpha without being lower or upper, such as "male and female ordinals" (\xAA and \xBA) in the @@ -114,24 +116,24 @@ test for alnum specially. */ memset(p, 0, cbit_length); for (i = 0; i < 256; i++) { - if (isdigit(i)) p[cbit_digit + i/8] |= 1u << (i&7); - if (isupper(i)) p[cbit_upper + i/8] |= 1u << (i&7); - if (islower(i)) p[cbit_lower + i/8] |= 1u << (i&7); - if (isalnum(i)) p[cbit_word + i/8] |= 1u << (i&7); - if (i == '_') p[cbit_word + i/8] |= 1u << (i&7); - if (isspace(i)) p[cbit_space + i/8] |= 1u << (i&7); - if (isxdigit(i))p[cbit_xdigit + i/8] |= 1u << (i&7); - if (isgraph(i)) p[cbit_graph + i/8] |= 1u << (i&7); - if (isprint(i)) p[cbit_print + i/8] |= 1u << (i&7); - if (ispunct(i)) p[cbit_punct + i/8] |= 1u << (i&7); - if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1u << (i&7); + if (isdigit(i)) p[cbit_digit + i/8] |= 1u << (i&7); + if (isupper(i)) p[cbit_upper + i/8] |= 1u << (i&7); + if (islower(i)) p[cbit_lower + i/8] |= 1u << (i&7); + if (isalnum(i)) p[cbit_word + i/8] |= 1u << (i&7); + if (i == '_') p[cbit_word + i/8] |= 1u << (i&7); + if (isspace(i)) p[cbit_space + i/8] |= 1u << (i&7); + if (isxdigit(i)) p[cbit_xdigit + i/8] |= 1u << (i&7); + if (isgraph(i)) p[cbit_graph + i/8] |= 1u << (i&7); + if (isprint(i)) p[cbit_print + i/8] |= 1u << (i&7); + if (ispunct(i)) p[cbit_punct + i/8] |= 1u << (i&7); + if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1u << (i&7); } p += cbit_length; /* Finally, the character type table. In this, we used to exclude VT from the white space chars, because Perl didn't recognize it as such for \s and for -comments within regexes. However, Perl changed at release 5.18, so PCRE changed -at release 8.34. */ +comments within regexes. However, Perl changed at release 5.18, so PCRE1 +changed at release 8.34 and it's always been this way for PCRE2. */ for (i = 0; i < 256; i++) { @@ -147,4 +149,15 @@ for (i = 0; i < 256; i++) return yield; } +#ifndef PCRE2_DFTABLES /* Compiling the library */ +PCRE2_EXP_DEFN void PCRE2_CALL_CONVENTION +pcre2_maketables_free(pcre2_general_context *gcontext, const uint8_t *tables) +{ + if (gcontext) + gcontext->memctl.free((void *)tables, gcontext->memctl.memory_data); + else + free((void *)tables); +} +#endif + /* End of pcre2_maketables.c */ diff --git a/dist2/src/pcre2_match.c b/dist2/src/pcre2_match.c index e5cf2c42..e3f78c2c 100644 --- a/dist2/src/pcre2_match.c +++ b/dist2/src/pcre2_match.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2015-2019 University of Cambridge + New API code Copyright (c) 2015-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -381,8 +381,12 @@ length = Fovector[offset+1] - Fovector[offset]; if (caseless) { #if defined SUPPORT_UNICODE - if ((mb->poptions & PCRE2_UTF) != 0) + BOOL utf = (mb->poptions & PCRE2_UTF) != 0; + + if (utf || (mb->poptions & PCRE2_UCP) != 0) { + PCRE2_SPTR endptr = p + length; + /* Match characters up to the end of the reference. NOTE: the number of code units matched may differ, because in UTF-8 there are some characters whose upper and lower case codes have different numbers of bytes. For @@ -390,16 +394,25 @@ if (caseless) bytes in UTF-8); a sequence of 3 of the former uses 6 bytes, as does a sequence of two of the latter. It is important, therefore, to check the length along the reference, not along the subject (earlier code did this - wrong). */ + wrong). UCP without uses Unicode properties but without UTF encoding. */ - PCRE2_SPTR endptr = p + length; while (p < endptr) { uint32_t c, d; const ucd_record *ur; if (eptr >= mb->end_subject) return 1; /* Partial match */ - GETCHARINC(c, eptr); - GETCHARINC(d, p); + + if (utf) + { + GETCHARINC(c, eptr); + GETCHARINC(d, p); + } + else + { + c = *eptr++; + d = *p++; + } + ur = GET_UCD(d); if (c != d && c != (uint32_t)((int)d + ur->other_case)) { @@ -415,8 +428,7 @@ if (caseless) else #endif - /* Not in UTF mode */ - + /* Not in UTF or UCP mode */ { for (; length > 0; length--) { @@ -433,7 +445,8 @@ if (caseless) } /* In the caseful case, we can just compare the code units, whether or not we -are in UTF mode. When partial matching, we have to do this unit-by-unit. */ +are in UTF and/or UCP mode. When partial matching, we have to do this unit by +unit. */ else { @@ -491,27 +504,32 @@ heap is used for a larger vector. *************************************************/ /* These macros pack up tests that are used for partial matching several times -in the code. We set the "hit end" flag if the pointer is at the end of the -subject and also past the earliest inspected character (i.e. something has been -matched, even if not part of the actual matched string). For hard partial -matching, we then return immediately. The second one is used when we already -know we are past the end of the subject. */ +in the code. The second one is used when we already know we are past the end of +the subject. We set the "hit end" flag if the pointer is at the end of the +subject and either (a) the pointer is past the earliest inspected character +(i.e. something has been matched, even if not part of the actual matched +string), or (b) the pattern contains a lookbehind. These are the conditions for +which adding more characters may allow the current match to continue. + +For hard partial matching, we immediately return a partial match. Otherwise, +carrying on means that a complete match on the current subject will be sought. +A partial match is returned only if no complete match can be found. */ #define CHECK_PARTIAL()\ - if (mb->partial != 0 && Feptr >= mb->end_subject && \ - Feptr > mb->start_used_ptr) \ + if (Feptr >= mb->end_subject) \ { \ - mb->hitend = TRUE; \ - if (mb->partial > 1) return PCRE2_ERROR_PARTIAL; \ + SCHECK_PARTIAL(); \ } #define SCHECK_PARTIAL()\ - if (mb->partial != 0 && Feptr > mb->start_used_ptr) \ + if (mb->partial != 0 && \ + (Feptr > mb->start_used_ptr || mb->allowemptypartial)) \ { \ mb->hitend = TRUE; \ if (mb->partial > 1) return PCRE2_ERROR_PARTIAL; \ } + /* These macros are used to implement backtracking. They simulate a recursive call to the match() function by means of a local vector of frames which remember the backtracking points. */ @@ -570,8 +588,8 @@ match(PCRE2_SPTR start_eptr, PCRE2_SPTR start_ecode, PCRE2_SIZE *ovector, heapframe *F; /* Current frame pointer */ heapframe *N = NULL; /* Temporary frame pointers */ heapframe *P = NULL; -heapframe *assert_accept_frame; /* For passing back the frame with captures */ -PCRE2_SIZE frame_copy_size; /* Amount to copy when creating a new frame */ +heapframe *assert_accept_frame = NULL; /* For passing back a frame with captures */ +PCRE2_SIZE frame_copy_size; /* Amount to copy when creating a new frame */ /* Local variables that do not need to be preserved over calls to RRMATCH(). */ @@ -594,12 +612,13 @@ BOOL condition; /* Used in conditional groups */ BOOL cur_is_word; /* Used in "word" tests */ BOOL prev_is_word; /* Used in "word" tests */ -/* UTF flag */ +/* UTF and UCP flags */ #ifdef SUPPORT_UNICODE BOOL utf = (mb->poptions & PCRE2_UTF) != 0; +BOOL ucp = (mb->poptions & PCRE2_UCP) != 0; #else -BOOL utf = FALSE; +BOOL utf = FALSE; /* Required for convenience even when no Unicode support */ #endif /* This is the length of the last part of a backtracking frame that must be @@ -924,6 +943,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); } else #endif + /* Not UTF mode */ { if (mb->end_subject - Feptr < 1) @@ -983,10 +1003,30 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (dc != fc && dc != UCD_OTHERCASE(fc)) RRETURN(MATCH_NOMATCH); } } + + /* If UCP is set without UTF we must do the same as above, but with one + character per code unit. */ + + else if (ucp) + { + uint32_t cc = UCHAR21(Feptr); + fc = Fecode[1]; + if (fc < 128) + { + if (mb->lcc[fc] != TABLE_GET(cc, mb->lcc, cc)) RRETURN(MATCH_NOMATCH); + } + else + { + if (cc != fc && cc != UCD_OTHERCASE(fc)) RRETURN(MATCH_NOMATCH); + } + Feptr++; + Fecode += 2; + } + else #endif /* SUPPORT_UNICODE */ - /* Not UTF mode; use the table for characters < 256. */ + /* Not UTF or UCP mode; use the table for characters < 256. */ { if (TABLE_GET(Fecode[1], mb->lcc, Fecode[1]) != TABLE_GET(*Feptr, mb->lcc, *Feptr)) RRETURN(MATCH_NOMATCH); @@ -1006,6 +1046,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); SCHECK_PARTIAL(); RRETURN(MATCH_NOMATCH); } + #ifdef SUPPORT_UNICODE if (utf) { @@ -1022,15 +1063,42 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (ch > 127) ch = UCD_OTHERCASE(ch); else - ch = TABLE_GET(ch, mb->fcc, ch); + ch = (mb->fcc)[ch]; if (ch == fc) RRETURN(MATCH_NOMATCH); } } + + /* UCP without UTF is as above, but with one character per code unit. */ + + else if (ucp) + { + uint32_t ch; + fc = UCHAR21INC(Feptr); + ch = Fecode[1]; + Fecode += 2; + + if (ch == fc) + { + RRETURN(MATCH_NOMATCH); /* Caseful match */ + } + else if (Fop == OP_NOTI) /* If caseless */ + { + if (ch > 127) + ch = UCD_OTHERCASE(ch); + else + ch = (mb->fcc)[ch]; + if (ch == fc) RRETURN(MATCH_NOMATCH); + } + } + else #endif /* SUPPORT_UNICODE */ + + /* Neither UTF nor UCP is set */ + { uint32_t ch = Fecode[1]; - fc = *Feptr++; + fc = UCHAR21INC(Feptr); if (ch == fc || (Fop == OP_NOTI && TABLE_GET(ch, mb->fcc, ch) == fc)) RRETURN(MATCH_NOMATCH); Fecode += 2; @@ -1240,7 +1308,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); #endif /* SUPPORT_UNICODE */ /* When not in UTF mode, load a single-code-unit character. Then proceed as - above. */ + above, using Unicode casing if either UTF or UCP is set. */ Lc = *Fecode++; @@ -1249,11 +1317,15 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (Fop >= OP_STARI) { #if PCRE2_CODE_UNIT_WIDTH == 8 - /* Lc must be < 128 in UTF-8 mode. */ +#ifdef SUPPORT_UNICODE + if (ucp && !utf && Lc > 127) Loc = UCD_OTHERCASE(Lc); + else +#endif /* SUPPORT_UNICODE */ + /* Lc will be < 128 in UTF-8 mode. */ Loc = mb->fcc[Lc]; #else /* 16-bit & 32-bit */ #ifdef SUPPORT_UNICODE - if (utf && Lc > 127) Loc = UCD_OTHERCASE(Lc); + if ((utf || ucp) && Lc > 127) Loc = UCD_OTHERCASE(Lc); else #endif /* SUPPORT_UNICODE */ Loc = TABLE_GET(Lc, mb->fcc, Lc); @@ -1486,7 +1558,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (Fop >= OP_NOTSTARI) /* Caseless */ { #ifdef SUPPORT_UNICODE - if (utf && Lc > 127) + if ((utf || ucp) && Lc > 127) Loc = UCD_OTHERCASE(Lc); else #endif /* SUPPORT_UNICODE */ @@ -5127,6 +5199,8 @@ fprintf(stderr, "++ op=%d\n", *Fecode); case OP_ASSERT: case OP_ASSERTBACK: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: Lframe_type = GF_NOCAPTURE | Fop; for (;;) { @@ -5412,7 +5486,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); { while (number-- > 0) { - if (Feptr <= mb->start_subject) RRETURN(MATCH_NOMATCH); + if (Feptr <= mb->check_subject) RRETURN(MATCH_NOMATCH); Feptr--; BACKCHAR(Feptr); } @@ -5420,7 +5494,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); else #endif - /* No UTF-8 support, or not in UTF-8 mode: count is byte count */ + /* No UTF-8 support, or not in UTF-8 mode: count is code unit count */ { if ((ptrdiff_t)number > Feptr - mb->start_subject) RRETURN(MATCH_NOMATCH); @@ -5472,15 +5546,16 @@ fprintf(stderr, "++ op=%d\n", *Fecode); /* If we are at the end of an assertion that is a condition, return a match, discarding any intermediate backtracking points. Copy back the - captures into the frame before N so that they are set on return. Doing - this for all assertions, both positive and negative, seems to match what - Perl does. */ + mark setting and the captures into the frame before N so that they are + set on return. Doing this for all assertions, both positive and negative, + seems to match what Perl does. */ if (GF_IDMASK(N->group_frame_type) == GF_CONDASSERT) { memcpy((char *)P + offsetof(heapframe, ovector), Fovector, Foffset_top * sizeof(PCRE2_SIZE)); P->offset_top = Foffset_top; + P->mark = Fmark; Fback_frame = (char *)F - (char *)P; RRETURN(MATCH_MATCH); } @@ -5496,10 +5571,20 @@ fprintf(stderr, "++ op=%d\n", *Fecode); case OP_SCOND: break; - /* Positive assertions are like OP_ONCE, except that in addition the + /* Non-atomic positive assertions are like OP_BRA, except that the subject pointer must be put back to where it was at the start of the assertion. */ + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: + if (Feptr > mb->last_used_ptr) mb->last_used_ptr = Feptr; + Feptr = P->eptr; + break; + + /* Atomic positive assertions are like OP_ONCE, except that in addition + the subject pointer must be put back to where it was at the start of the + assertion. */ + case OP_ASSERT: case OP_ASSERTBACK: if (Feptr > mb->last_used_ptr) mb->last_used_ptr = Feptr; @@ -5640,7 +5725,11 @@ fprintf(stderr, "++ op=%d\n", *Fecode); case OP_EOD: if (Feptr < mb->end_subject) RRETURN(MATCH_NOMATCH); - SCHECK_PARTIAL(); + if (mb->partial != 0) + { + mb->hitend = TRUE; + if (mb->partial > 1) return PCRE2_ERROR_PARTIAL; + } Fecode++; break; @@ -5665,7 +5754,11 @@ fprintf(stderr, "++ op=%d\n", *Fecode); /* Either at end of string or \n before end. */ - SCHECK_PARTIAL(); + if (mb->partial != 0) + { + mb->hitend = TRUE; + if (mb->partial > 1) return PCRE2_ERROR_PARTIAL; + } Fecode++; break; @@ -5743,7 +5836,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); case OP_NOT_WORD_BOUNDARY: case OP_WORD_BOUNDARY: - if (Feptr == mb->start_subject) prev_is_word = FALSE; else + if (Feptr == mb->check_subject) prev_is_word = FALSE; else { PCRE2_SPTR lastptr = Feptr - 1; #ifdef SUPPORT_UNICODE @@ -5946,6 +6039,7 @@ in rrc. */ #define LBL(val) case val: goto L_RM##val; RETURN_SWITCH: +if (Feptr > mb->last_used_ptr) mb->last_used_ptr = Feptr; if (Frdepth == 0) return rrc; /* Exit from the top level */ F = (heapframe *)((char *)F - Fback_frame); /* Backtrack */ mb->cb->callout_flags |= PCRE2_CALLOUT_BACKTRACK; /* Note for callouts */ @@ -5999,9 +6093,9 @@ Arguments: Returns: > 0 => success; value is the number of ovector pairs filled = 0 => success, but ovector is not big enough - -1 => failed to match (PCRE2_ERROR_NOMATCH) - -2 => partial match (PCRE2_ERROR_PARTIAL) - < -2 => some kind of unexpected problem + = -1 => failed to match (PCRE2_ERROR_NOMATCH) + = -2 => partial match (PCRE2_ERROR_PARTIAL) + < -2 => some kind of unexpected problem */ PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION @@ -6014,13 +6108,16 @@ int was_zero_terminated = 0; const uint8_t *start_bits = NULL; const pcre2_real_code *re = (const pcre2_real_code *)code; - BOOL anchored; BOOL firstline; BOOL has_first_cu = FALSE; BOOL has_req_cu = FALSE; BOOL startline; -BOOL utf; + +#if PCRE2_CODE_UNIT_WIDTH == 8 +BOOL memchr_not_found_first_cu; +BOOL memchr_not_found_first_cu2; +#endif PCRE2_UCHAR first_cu = 0; PCRE2_UCHAR first_cu2 = 0; @@ -6029,10 +6126,29 @@ PCRE2_UCHAR req_cu2 = 0; PCRE2_SPTR bumpalong_limit; PCRE2_SPTR end_subject; +PCRE2_SPTR true_end_subject; PCRE2_SPTR start_match = subject + start_offset; PCRE2_SPTR req_cu_ptr = start_match - 1; -PCRE2_SPTR start_partial = NULL; -PCRE2_SPTR match_partial = NULL; +PCRE2_SPTR start_partial; +PCRE2_SPTR match_partial; + +#ifdef SUPPORT_JIT +BOOL use_jit; +#endif + +/* This flag is needed even when Unicode is not supported for convenience +(it is used by the IS_NEWLINE macro). */ + +BOOL utf = FALSE; + +#ifdef SUPPORT_UNICODE +BOOL ucp = FALSE; +BOOL allow_invalid; +uint32_t fragment_options = 0; +#ifdef SUPPORT_JIT +BOOL jit_checked_utf = FALSE; +#endif +#endif /* SUPPORT_UNICODE */ PCRE2_SIZE frame_size; @@ -6060,7 +6176,7 @@ if (length == PCRE2_ZERO_TERMINATED) length = PRIV(strlen)(subject); was_zero_terminated = 1; } -end_subject = subject + length; +true_end_subject = end_subject = subject + length; /* Plausibility checks */ @@ -6096,12 +6212,25 @@ options |= (re->flags & FF) / ((FF & (~FF+1)) / (OO & (~OO+1))); #undef FF #undef OO -/* These two settings are used in the code for checking a UTF string that -follows immediately afterwards. Other values in the mb block are used only -during interpretive processing, not when the JIT support is in use, so they are -set up later. */ +/* If the pattern was successfully studied with JIT support, we will run the +JIT executable instead of the rest of this function. Most options must be set +at compile time for the JIT code to be usable. */ +#ifdef SUPPORT_JIT +use_jit = (re->executable_jit != NULL && + (options & ~PUBLIC_JIT_MATCH_OPTIONS) == 0); +#endif + +/* Initialize UTF/UCP parameters. */ + +#ifdef SUPPORT_UNICODE utf = (re->overall_options & PCRE2_UTF) != 0; +allow_invalid = (re->overall_options & PCRE2_MATCH_INVALID_UTF) != 0; +ucp = (re->overall_options & PCRE2_UCP) != 0; +#endif /* SUPPORT_UNICODE */ + +/* Convert the partial matching flags into an integer. */ + mb->partial = ((options & PCRE2_PARTIAL_HARD) != 0)? 2 : ((options & PCRE2_PARTIAL_SOFT) != 0)? 1 : 0; @@ -6112,88 +6241,107 @@ if (mb->partial != 0 && ((re->overall_options | options) & PCRE2_ENDANCHORED) != 0) return PCRE2_ERROR_BADOPTION; -/* Check a UTF string for validity if required. For 8-bit and 16-bit strings, -we must also check that a starting offset does not point into the middle of a -multiunit character. We check only the portion of the subject that is going to -be inspected during matching - from the offset minus the maximum back reference -to the given length. This saves time when a small part of a large subject is -being matched by the use of a starting offset. Note that the maximum lookbehind -is a number of characters, not code units. */ +/* It is an error to set an offset limit without setting the flag at compile +time. */ -#ifdef SUPPORT_UNICODE -if (utf && (options & PCRE2_NO_UTF_CHECK) == 0) +if (mcontext != NULL && mcontext->offset_limit != PCRE2_UNSET && + (re->overall_options & PCRE2_USE_OFFSET_LIMIT) == 0) + return PCRE2_ERROR_BADOFFSETLIMIT; + +/* If the match data block was previously used with PCRE2_COPY_MATCHED_SUBJECT, +free the memory that was obtained. Set the field to NULL for no match cases. */ + +if ((match_data->flags & PCRE2_MD_COPIED_SUBJECT) != 0) { - PCRE2_SPTR check_subject = start_match; /* start_match includes offset */ + match_data->memctl.free((void *)match_data->subject, + match_data->memctl.memory_data); + match_data->flags &= ~PCRE2_MD_COPIED_SUBJECT; + } +match_data->subject = NULL; + +/* Zero the error offset in case the first code unit is invalid UTF. */ + +match_data->startchar = 0; + + +/* ============================= JIT matching ============================== */ + +/* Prepare for JIT matching. Check a UTF string for validity unless no check is +requested or invalid UTF can be handled. We check only the portion of the +subject that might be be inspected during matching - from the offset minus the +maximum lookbehind to the given length. This saves time when a small part of a +large subject is being matched by the use of a starting offset. Note that the +maximum lookbehind is a number of characters, not code units. */ - if (start_offset > 0) +#ifdef SUPPORT_JIT +if (use_jit) + { +#ifdef SUPPORT_UNICODE + if (utf && (options & PCRE2_NO_UTF_CHECK) == 0 && !allow_invalid) { #if PCRE2_CODE_UNIT_WIDTH != 32 unsigned int i; +#endif + + /* For 8-bit and 16-bit UTF, check that the first code unit is a valid + character start. */ + +#if PCRE2_CODE_UNIT_WIDTH != 32 if (start_match < end_subject && NOT_FIRSTCU(*start_match)) - return PCRE2_ERROR_BADUTFOFFSET; - for (i = re->max_lookbehind; i > 0 && check_subject > subject; i--) { - check_subject--; - while (check_subject > subject && + if (start_offset > 0) return PCRE2_ERROR_BADUTFOFFSET; #if PCRE2_CODE_UNIT_WIDTH == 8 - (*check_subject & 0xc0) == 0x80) + return PCRE2_ERROR_UTF8_ERR20; /* Isolated 0x80 byte */ +#else + return PCRE2_ERROR_UTF16_ERR3; /* Isolated low surrogate */ +#endif + } +#endif /* WIDTH != 32 */ + + /* Move back by the maximum lookbehind, just in case it happens at the very + start of matching. */ + +#if PCRE2_CODE_UNIT_WIDTH != 32 + for (i = re->max_lookbehind; i > 0 && start_match > subject; i--) + { + start_match--; + while (start_match > subject && +#if PCRE2_CODE_UNIT_WIDTH == 8 + (*start_match & 0xc0) == 0x80) #else /* 16-bit */ - (*check_subject & 0xfc00) == 0xdc00) -#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ - check_subject--; + (*start_match & 0xfc00) == 0xdc00) +#endif + start_match--; } -#else +#else /* PCRE2_CODE_UNIT_WIDTH != 32 */ + /* In the 32-bit library, one code unit equals one character. However, we cannot just subtract the lookbehind and then compare pointers, because a very large lookbehind could create an invalid pointer. */ if (start_offset >= re->max_lookbehind) - check_subject -= re->max_lookbehind; + start_match -= re->max_lookbehind; else - check_subject = subject; + start_match = subject; #endif /* PCRE2_CODE_UNIT_WIDTH != 32 */ - } - /* Validate the relevant portion of the subject. After an error, adjust the - offset to be an absolute offset in the whole string. */ + /* Validate the relevant portion of the subject. Adjust the offset of an + invalid code point to be an absolute offset in the whole string. */ - match_data->rc = PRIV(valid_utf)(check_subject, - length - (check_subject - subject), &(match_data->startchar)); - if (match_data->rc != 0) - { - match_data->startchar += check_subject - subject; - return match_data->rc; + match_data->rc = PRIV(valid_utf)(start_match, + length - (start_match - subject), &(match_data->startchar)); + if (match_data->rc != 0) + { + match_data->startchar += start_match - subject; + return match_data->rc; + } + jit_checked_utf = TRUE; } - } #endif /* SUPPORT_UNICODE */ -/* It is an error to set an offset limit without setting the flag at compile -time. */ - -if (mcontext != NULL && mcontext->offset_limit != PCRE2_UNSET && - (re->overall_options & PCRE2_USE_OFFSET_LIMIT) == 0) - return PCRE2_ERROR_BADOFFSETLIMIT; - -/* If the match data block was previously used with PCRE2_COPY_MATCHED_SUBJECT, -free the memory that was obtained. Set the field to NULL for no match cases. */ - -if ((match_data->flags & PCRE2_MD_COPIED_SUBJECT) != 0) - { - match_data->memctl.free((void *)match_data->subject, - match_data->memctl.memory_data); - match_data->flags &= ~PCRE2_MD_COPIED_SUBJECT; - } -match_data->subject = NULL; - -/* If the pattern was successfully studied with JIT support, run the JIT -executable instead of the rest of this function. Most options must be set at -compile time for the JIT code to be usable. Fallback to the normal code path if -an unsupported option is set or if JIT returns BADOPTION (which means that the -selected normal or partial matching mode was not compiled). */ + /* If JIT returns BADOPTION, which means that the selected complete or + partial matching mode was not compiled, fall through to the interpreter. */ -#ifdef SUPPORT_JIT -if (re->executable_jit != NULL && (options & ~PUBLIC_JIT_MATCH_OPTIONS) == 0) - { rc = pcre2_jit_match(code, subject, length, start_offset, options, match_data, mcontext); if (rc != PCRE2_ERROR_JIT_BADOPTION) @@ -6210,10 +6358,152 @@ if (re->executable_jit != NULL && (options & ~PUBLIC_JIT_MATCH_OPTIONS) == 0) return rc; } } +#endif /* SUPPORT_JIT */ + +/* ========================= End of JIT matching ========================== */ + + +/* Proceed with non-JIT matching. The default is to allow lookbehinds to the +start of the subject. A UTF check when there is a non-zero offset may change +this. */ + +mb->check_subject = subject; + +/* If a UTF subject string was not checked for validity in the JIT code above, +check it here, and handle support for invalid UTF strings. The check above +happens only when invalid UTF is not supported and PCRE2_NO_CHECK_UTF is unset. +If we get here in those circumstances, it means the subject string is valid, +but for some reason JIT matching was not successful. There is no need to check +the subject again. + +We check only the portion of the subject that might be be inspected during +matching - from the offset minus the maximum lookbehind to the given length. +This saves time when a small part of a large subject is being matched by the +use of a starting offset. Note that the maximum lookbehind is a number of +characters, not code units. + +Note also that support for invalid UTF forces a check, overriding the setting +of PCRE2_NO_CHECK_UTF. */ + +#ifdef SUPPORT_UNICODE +if (utf && +#ifdef SUPPORT_JIT + !jit_checked_utf && +#endif + ((options & PCRE2_NO_UTF_CHECK) == 0 || allow_invalid)) + { +#if PCRE2_CODE_UNIT_WIDTH != 32 + BOOL skipped_bad_start = FALSE; #endif -/* Carry on with non-JIT matching. A NULL match context means "use a default -context", but we take the memory control functions from the pattern. */ + /* For 8-bit and 16-bit UTF, check that the first code unit is a valid + character start. If we are handling invalid UTF, just skip over such code + units. Otherwise, give an appropriate error. */ + +#if PCRE2_CODE_UNIT_WIDTH != 32 + if (allow_invalid) + { + while (start_match < end_subject && NOT_FIRSTCU(*start_match)) + { + start_match++; + skipped_bad_start = TRUE; + } + } + else if (start_match < end_subject && NOT_FIRSTCU(*start_match)) + { + if (start_offset > 0) return PCRE2_ERROR_BADUTFOFFSET; +#if PCRE2_CODE_UNIT_WIDTH == 8 + return PCRE2_ERROR_UTF8_ERR20; /* Isolated 0x80 byte */ +#else + return PCRE2_ERROR_UTF16_ERR3; /* Isolated low surrogate */ +#endif + } +#endif /* WIDTH != 32 */ + + /* The mb->check_subject field points to the start of UTF checking; + lookbehinds can go back no further than this. */ + + mb->check_subject = start_match; + + /* Move back by the maximum lookbehind, just in case it happens at the very + start of matching, but don't do this if we skipped bad 8-bit or 16-bit code + units above. */ + +#if PCRE2_CODE_UNIT_WIDTH != 32 + if (!skipped_bad_start) + { + unsigned int i; + for (i = re->max_lookbehind; i > 0 && mb->check_subject > subject; i--) + { + mb->check_subject--; + while (mb->check_subject > subject && +#if PCRE2_CODE_UNIT_WIDTH == 8 + (*mb->check_subject & 0xc0) == 0x80) +#else /* 16-bit */ + (*mb->check_subject & 0xfc00) == 0xdc00) +#endif + mb->check_subject--; + } + } +#else /* PCRE2_CODE_UNIT_WIDTH != 32 */ + + /* In the 32-bit library, one code unit equals one character. However, + we cannot just subtract the lookbehind and then compare pointers, because + a very large lookbehind could create an invalid pointer. */ + + if (start_offset >= re->max_lookbehind) + mb->check_subject -= re->max_lookbehind; + else + mb->check_subject = subject; +#endif /* PCRE2_CODE_UNIT_WIDTH != 32 */ + + /* Validate the relevant portion of the subject. There's a loop in case we + encounter bad UTF in the characters preceding start_match which we are + scanning because of a lookbehind. */ + + for (;;) + { + match_data->rc = PRIV(valid_utf)(mb->check_subject, + length - (mb->check_subject - subject), &(match_data->startchar)); + + if (match_data->rc == 0) break; /* Valid UTF string */ + + /* Invalid UTF string. Adjust the offset to be an absolute offset in the + whole string. If we are handling invalid UTF strings, set end_subject to + stop before the bad code unit, and set the options to "not end of line". + Otherwise return the error. */ + + match_data->startchar += mb->check_subject - subject; + if (!allow_invalid || match_data->rc > 0) return match_data->rc; + end_subject = subject + match_data->startchar; + + /* If the end precedes start_match, it means there is invalid UTF in the + extra code units we reversed over because of a lookbehind. Advance past the + first bad code unit, and then skip invalid character starting code units in + 8-bit and 16-bit modes, and try again. */ + + if (end_subject < start_match) + { + mb->check_subject = end_subject + 1; +#if PCRE2_CODE_UNIT_WIDTH != 32 + while (mb->check_subject < start_match && NOT_FIRSTCU(*mb->check_subject)) + mb->check_subject++; +#endif + } + + /* Otherwise, set the not end of line option, and do the match. */ + + else + { + fragment_options = PCRE2_NOTEOL; + break; + } + } + } +#endif /* SUPPORT_UNICODE */ + +/* A NULL match context means "use a default context", but we take the memory +control functions from the pattern. */ if (mcontext == NULL) { @@ -6225,8 +6515,8 @@ else mb->memctl = mcontext->memctl; anchored = ((re->overall_options | options) & PCRE2_ANCHORED) != 0; firstline = (re->overall_options & PCRE2_FIRSTLINE) != 0; startline = (re->flags & PCRE2_STARTLINE) != 0; -bumpalong_limit = (mcontext->offset_limit == PCRE2_UNSET)? - end_subject : subject + mcontext->offset_limit; +bumpalong_limit = (mcontext->offset_limit == PCRE2_UNSET)? + true_end_subject : subject + mcontext->offset_limit; /* Initialize and set up the fixed fields in the callout block, with a pointer in the match block. */ @@ -6237,7 +6527,8 @@ cb.subject = subject; cb.subject_length = (PCRE2_SIZE)(end_subject - subject); cb.callout_flags = 0; -/* Fill in the remaining fields in the match block. */ +/* Fill in the remaining fields in the match block, except for moptions, which +gets set later. */ mb->callout = mcontext->callout; mb->callout_data = mcontext->callout_data; @@ -6246,13 +6537,11 @@ mb->start_subject = subject; mb->start_offset = start_offset; mb->end_subject = end_subject; mb->hasthen = (re->flags & PCRE2_HASTHEN) != 0; - -mb->moptions = options; /* Match options */ -mb->poptions = re->overall_options; /* Pattern options */ - +mb->allowemptypartial = (re->max_lookbehind > 0) || + (re->flags & PCRE2_MATCH_EMPTY) != 0; +mb->poptions = re->overall_options; /* Pattern options */ mb->ignore_skip_arg = 0; -mb->mark = mb->nomatch_mark = NULL; /* In case never set */ -mb->hitend = FALSE; +mb->mark = mb->nomatch_mark = NULL; /* In case never set */ /* The name table is needed for finding all the numbers associated with a given name, for condition testing. The code follows the name table. */ @@ -6375,9 +6664,13 @@ if ((re->flags & PCRE2_FIRSTSET) != 0) if ((re->flags & PCRE2_FIRSTCASELESS) != 0) { first_cu2 = TABLE_GET(first_cu, mb->fcc, first_cu); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (utf && first_cu > 127) first_cu2 = UCD_OTHERCASE(first_cu); +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (first_cu > 127 && ucp && !utf) first_cu2 = UCD_OTHERCASE(first_cu); +#else + if (first_cu > 127 && (utf || ucp)) first_cu2 = UCD_OTHERCASE(first_cu); #endif +#endif /* SUPPORT_UNICODE */ } } else @@ -6393,9 +6686,13 @@ if ((re->flags & PCRE2_LASTSET) != 0) if ((re->flags & PCRE2_LASTCASELESS) != 0) { req_cu2 = TABLE_GET(req_cu, mb->fcc, req_cu); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 8 - if (utf && req_cu > 127) req_cu2 = UCD_OTHERCASE(req_cu); +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (req_cu > 127 && ucp && !utf) req_cu2 = UCD_OTHERCASE(req_cu); +#else + if (req_cu > 127 && (utf || ucp)) req_cu2 = UCD_OTHERCASE(req_cu); #endif +#endif /* SUPPORT_UNICODE */ } } @@ -6405,6 +6702,18 @@ if ((re->flags & PCRE2_LASTSET) != 0) /* Loop for handling unanchored repeated matching attempts; for anchored regexs the loop runs just once. */ +#ifdef SUPPORT_UNICODE +FRAGMENT_RESTART: +#endif + +start_partial = match_partial = NULL; +mb->hitend = FALSE; + +#if PCRE2_CODE_UNIT_WIDTH == 8 +memchr_not_found_first_cu = FALSE; +memchr_not_found_first_cu2 = FALSE; +#endif + for(;;) { PCRE2_SPTR new_start_match; @@ -6474,7 +6783,10 @@ for(;;) /* Not anchored. Advance to a unique first code unit if there is one. In 8-bit mode, the use of memchr() gives a big speed up, even though we have to call it twice in caseless mode, in order to find the earliest occurrence - of the character in either of its cases. */ + of the character in either of its cases. If a call to memchr() that + searches the rest of the subject fails to find one case, remember that in + order not to keep on repeating the search. This can make a huge difference + when the strings are very long and only one case is present. */ else { @@ -6488,11 +6800,29 @@ for(;;) (smc = UCHAR21TEST(start_match)) != first_cu && smc != first_cu2) start_match++; + #else /* 8-bit code units */ - PCRE2_SPTR pp1 = - memchr(start_match, first_cu, end_subject-start_match); - PCRE2_SPTR pp2 = - memchr(start_match, first_cu2, end_subject-start_match); + PCRE2_SPTR pp1 = NULL; + PCRE2_SPTR pp2 = NULL; + PCRE2_SIZE cu2size = end_subject - start_match; + + if (!memchr_not_found_first_cu) + { + pp1 = memchr(start_match, first_cu, end_subject - start_match); + if (pp1 == NULL) memchr_not_found_first_cu = TRUE; + else cu2size = pp1 - start_match; + } + + /* If pp1 is not NULL, we have arranged to search only as far as pp1, + to see if the other case is earlier, so we can set "not found" only + when both searches have returned NULL. */ + + if (!memchr_not_found_first_cu2) + { + pp2 = memchr(start_match, first_cu2, cu2size); + memchr_not_found_first_cu2 = (pp2 == NULL && pp1 == NULL); + } + if (pp1 == NULL) start_match = (pp2 == NULL)? end_subject : pp2; else @@ -6514,17 +6844,18 @@ for(;;) #endif } - /* If we can't find the required code unit, having reached the true end - of the subject, break the bumpalong loop, to force a match failure, - except when doing partial matching, when we let the next cycle run at - the end of the subject. To see why, consider the pattern /(?<=abc)def/, - which partially matches "abc", even though the string does not contain - the starting character "d". If we have not reached the true end of the - subject (PCRE2_FIRSTLINE caused end_subject to be temporarily modified) - we also let the cycle run, because the matching string is legitimately - allowed to start with the first code unit of a newline. */ - - if (!mb->partial && start_match >= mb->end_subject) + /* If we can't find the required first code unit, having reached the + true end of the subject, break the bumpalong loop, to force a match + failure, except when doing partial matching, when we let the next cycle + run at the end of the subject. To see why, consider the pattern + /(?<=abc)def/, which partially matches "abc", even though the string + does not contain the starting character "d". If we have not reached the + true end of the subject (PCRE2_FIRSTLINE caused end_subject to be + temporarily modified) we also let the cycle run, because the matching + string is legitimately allowed to start with the first code unit of a + newline. */ + + if (mb->partial == 0 && start_match >= mb->end_subject) { rc = MATCH_NOMATCH; break; @@ -6583,7 +6914,7 @@ for(;;) /* See comment above in first_cu checking about the next few lines. */ - if (!mb->partial && start_match >= mb->end_subject) + if (mb->partial == 0 && start_match >= mb->end_subject) { rc = MATCH_NOMATCH; break; @@ -6597,8 +6928,10 @@ for(;;) /* The following two optimizations must be disabled for partial matching. */ - if (!mb->partial) + if (mb->partial == 0) { + PCRE2_SPTR p; + /* The minimum matching length is a lower bound; no string of that length may actually match the pattern. Although the value is, strictly, in characters, we treat it as code units to avoid spending too much time in @@ -6622,60 +6955,57 @@ for(;;) memchr() twice in the caseless case because we only need to check for the presence of the character in either case, not find the first occurrence. + The search can be skipped if the code unit was found later than the + current starting point in a previous iteration of the bumpalong loop. + HOWEVER: when the subject string is very, very long, searching to its end can take a long time, and give bad performance on quite ordinary - patterns. This showed up when somebody was matching something like - /^\d+C/ on a 32-megabyte string... so we don't do this when the string is - sufficiently long. */ + anchored patterns. This showed up when somebody was matching something + like /^\d+C/ on a 32-megabyte string... so we don't do this when the + string is sufficiently long, but it's worth searching a lot more for + unanchored patterns. */ - if (has_req_cu && end_subject - start_match < REQ_CU_MAX) + p = start_match + (has_first_cu? 1:0); + if (has_req_cu && p > req_cu_ptr) { - PCRE2_SPTR p = start_match + (has_first_cu? 1:0); - - /* We don't need to repeat the search if we haven't yet reached the - place we found it last time round the bumpalong loop. */ + PCRE2_SIZE check_length = end_subject - start_match; - if (p > req_cu_ptr) + if (check_length < REQ_CU_MAX || + (!anchored && check_length < REQ_CU_MAX * 1000)) { - if (p < end_subject) + if (req_cu != req_cu2) /* Caseless */ { - if (req_cu != req_cu2) /* Caseless */ - { #if PCRE2_CODE_UNIT_WIDTH != 8 - do - { - uint32_t pp = UCHAR21INCTEST(p); - if (pp == req_cu || pp == req_cu2) { p--; break; } - } - while (p < end_subject); - + while (p < end_subject) + { + uint32_t pp = UCHAR21INCTEST(p); + if (pp == req_cu || pp == req_cu2) { p--; break; } + } #else /* 8-bit code units */ - PCRE2_SPTR pp = p; - p = memchr(pp, req_cu, end_subject - pp); - if (p == NULL) - { - p = memchr(pp, req_cu2, end_subject - pp); - if (p == NULL) p = end_subject; - } -#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */ + PCRE2_SPTR pp = p; + p = memchr(pp, req_cu, end_subject - pp); + if (p == NULL) + { + p = memchr(pp, req_cu2, end_subject - pp); + if (p == NULL) p = end_subject; } +#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */ + } - /* The caseful case */ + /* The caseful case */ - else - { + else + { #if PCRE2_CODE_UNIT_WIDTH != 8 - do - { - if (UCHAR21INCTEST(p) == req_cu) { p--; break; } - } - while (p < end_subject); + while (p < end_subject) + { + if (UCHAR21INCTEST(p) == req_cu) { p--; break; } + } #else /* 8-bit code units */ - p = memchr(p, req_cu, end_subject - p); - if (p == NULL) p = end_subject; + p = memchr(p, req_cu, end_subject - p); + if (p == NULL) p = end_subject; #endif - } } /* If we can't find the required code unit, break the bumpalong loop, @@ -6715,6 +7045,11 @@ for(;;) mb->start_used_ptr = start_match; mb->last_used_ptr = start_match; +#ifdef SUPPORT_UNICODE + mb->moptions = options | fragment_options; +#else + mb->moptions = options; +#endif mb->match_call_count = 0; mb->end_offset_top = 0; mb->skip_arg_count = 0; @@ -6840,6 +7175,69 @@ for(;;) ENDLOOP: +/* If end_subject != true_end_subject, it means we are handling invalid UTF, +and have just processed a non-terminal fragment. If this resulted in no match +or a partial match we must carry on to the next fragment (a partial match is +returned to the caller only at the very end of the subject). A loop is used to +avoid trying to match against empty fragments; if the pattern can match an +empty string it would have done so already. */ + +#ifdef SUPPORT_UNICODE +if (utf && end_subject != true_end_subject && + (rc == MATCH_NOMATCH || rc == PCRE2_ERROR_PARTIAL)) + { + for (;;) + { + /* Advance past the first bad code unit, and then skip invalid character + starting code units in 8-bit and 16-bit modes. */ + + start_match = end_subject + 1; + +#if PCRE2_CODE_UNIT_WIDTH != 32 + while (start_match < true_end_subject && NOT_FIRSTCU(*start_match)) + start_match++; +#endif + + /* If we have hit the end of the subject, there isn't another non-empty + fragment, so give up. */ + + if (start_match >= true_end_subject) + { + rc = MATCH_NOMATCH; /* In case it was partial */ + break; + } + + /* Check the rest of the subject */ + + mb->check_subject = start_match; + rc = PRIV(valid_utf)(start_match, length - (start_match - subject), + &(match_data->startchar)); + + /* The rest of the subject is valid UTF. */ + + if (rc == 0) + { + mb->end_subject = end_subject = true_end_subject; + fragment_options = PCRE2_NOTBOL; + goto FRAGMENT_RESTART; + } + + /* A subsequent UTF error has been found; if the next fragment is + non-empty, set up to process it. Otherwise, let the loop advance. */ + + else if (rc < 0) + { + mb->end_subject = end_subject = start_match + match_data->startchar; + if (end_subject > start_match) + { + fragment_options = PCRE2_NOTBOL|PCRE2_NOTEOL; + goto FRAGMENT_RESTART; + } + } + } + } +#endif /* SUPPORT_UNICODE */ + /* Release an enlarged frame vector that is on the heap. */ if (mb->match_frames != mb->stack_frames) diff --git a/dist2/src/pcre2_match_data.c b/dist2/src/pcre2_match_data.c index ccc5f674..53e46987 100644 --- a/dist2/src/pcre2_match_data.c +++ b/dist2/src/pcre2_match_data.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -150,4 +150,17 @@ pcre2_get_startchar(pcre2_match_data *match_data) return match_data->startchar; } + + +/************************************************* +* Get size of match data block * +*************************************************/ + +PCRE2_EXP_DEFN PCRE2_SIZE PCRE2_CALL_CONVENTION +pcre2_get_match_data_size(pcre2_match_data *match_data) +{ +return offsetof(pcre2_match_data, ovector) + + 2 * (match_data->oveccount) * sizeof(PCRE2_SIZE); +} + /* End of pcre2_match_data.c */ diff --git a/dist2/src/pcre2_printint.c b/dist2/src/pcre2_printint.c index b132d44f..b9bab025 100644 --- a/dist2/src/pcre2_printint.c +++ b/dist2/src/pcre2_printint.c @@ -392,6 +392,8 @@ for(;;) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: case OP_ONCE: case OP_SCRIPT_RUN: case OP_COND: diff --git a/dist2/src/pcre2_serialize.c b/dist2/src/pcre2_serialize.c index cec1a035..ba17a26d 100644 --- a/dist2/src/pcre2_serialize.c +++ b/dist2/src/pcre2_serialize.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -90,7 +90,7 @@ if (codes == NULL || serialized_bytes == NULL || serialized_size == NULL) if (number_of_codes <= 0) return PCRE2_ERROR_BADDATA; /* Compute total size. */ -total_size = sizeof(pcre2_serialized_data) + tables_length; +total_size = sizeof(pcre2_serialized_data) + TABLES_LENGTH; tables = NULL; for (i = 0; i < number_of_codes; i++) @@ -121,8 +121,8 @@ data->number_of_codes = number_of_codes; /* Copy all compiled code data. */ dst_bytes = bytes + sizeof(pcre2_serialized_data); -memcpy(dst_bytes, tables, tables_length); -dst_bytes += tables_length; +memcpy(dst_bytes, tables, TABLES_LENGTH); +dst_bytes += TABLES_LENGTH; for (i = 0; i < number_of_codes; i++) { @@ -189,12 +189,12 @@ src_bytes = bytes + sizeof(pcre2_serialized_data); /* Decode tables. The reference count for the tables is stored immediately following them. */ -tables = memctl->malloc(tables_length + sizeof(PCRE2_SIZE), memctl->memory_data); +tables = memctl->malloc(TABLES_LENGTH + sizeof(PCRE2_SIZE), memctl->memory_data); if (tables == NULL) return PCRE2_ERROR_NOMEMORY; -memcpy(tables, src_bytes, tables_length); -*(PCRE2_SIZE *)(tables + tables_length) = number_of_codes; -src_bytes += tables_length; +memcpy(tables, src_bytes, TABLES_LENGTH); +*(PCRE2_SIZE *)(tables + TABLES_LENGTH) = number_of_codes; +src_bytes += TABLES_LENGTH; /* Decode the byte stream. We must not try to read the size from the compiled code block in the stream, because it might be unaligned, which causes errors on diff --git a/dist2/src/pcre2_study.c b/dist2/src/pcre2_study.c index e883c2eb..9bbb3757 100644 --- a/dist2/src/pcre2_study.c +++ b/dist2/src/pcre2_study.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -58,7 +58,7 @@ collecting data (e.g. minimum matching length). */ /* Returns from set_start_bits() */ -enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE, SSB_UNKNOWN }; +enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE, SSB_UNKNOWN, SSB_TOODEEP }; /************************************************* @@ -88,11 +88,13 @@ Arguments: countptr pointer to call count (to catch over complexity) backref_cache vector for caching back references. +This function is no longer called when the pattern contains (*ACCEPT); however, +the old code for returning -1 is retained, just in case. + Returns: the minimum length -1 \C in UTF-8 mode or (*ACCEPT) or pattern too complicated - or back reference to duplicate name/number -2 internal error (missing capturing bracket) -3 internal error (opcode not listed) */ @@ -103,6 +105,7 @@ find_minlength(const pcre2_real_code *re, PCRE2_SPTR code, int *backref_cache) { int length = -1; +int branchlength = 0; int prev_cap_recno = -1; int prev_cap_d = 0; int prev_recurse_recno = -1; @@ -110,9 +113,9 @@ int prev_recurse_d = 0; uint32_t once_fudge = 0; BOOL had_recurse = FALSE; BOOL dupcapused = (re->flags & PCRE2_DUPCAPUSED) != 0; -recurse_check this_recurse; -int branchlength = 0; +PCRE2_SPTR nextbranch = code + GET(code, 1); PCRE2_UCHAR *cc = (PCRE2_UCHAR *)code + 1 + LINK_SIZE; +recurse_check this_recurse; /* If this is a "could be empty" group, its minimum length is 0. */ @@ -128,16 +131,20 @@ if ((*countptr)++ > 1000) return -1; /* Scan along the opcodes for this branch. If we get to the end of the branch, check the length against that of the other branches. If the accumulated length -passes 16-bits, stop. */ +passes 16-bits, reset to that value and skip the rest of the branch. */ for (;;) { int d, min, recno; - PCRE2_UCHAR *cs, *ce; - PCRE2_UCHAR op = *cc; + PCRE2_UCHAR op, *cs, *ce; - if (branchlength >= UINT16_MAX) return UINT16_MAX; + if (branchlength >= UINT16_MAX) + { + branchlength = UINT16_MAX; + cc = (PCRE2_UCHAR *)nextbranch; + } + op = *cc; switch (op) { case OP_COND: @@ -206,7 +213,9 @@ for (;;) cc += 1 + LINK_SIZE; break; - /* ACCEPT makes things far too complicated; we have to give up. */ + /* ACCEPT makes things far too complicated; we have to give up. In fact, + from 10.34 onwards, if a pattern contains (*ACCEPT), this function is not + used. However, leave the code in place, just in case. */ case OP_ACCEPT: case OP_ASSERT_ACCEPT: @@ -214,9 +223,9 @@ for (;;) /* Reached end of a branch; if it's a ket it is the end of a nested call. If it's ALT it is an alternation in a nested call. If it is END it's - the end of the outer call. All can be handled by the same code. If an - ACCEPT was previously encountered, use the length that was in force at that - time, and pass back the shortest ACCEPT length. */ + the end of the outer call. All can be handled by the same code. If the + length of any branch is zero, there is no need to scan any subsequent + branches. */ case OP_ALT: case OP_KET: @@ -226,7 +235,8 @@ for (;;) case OP_END: if (length < 0 || (!had_recurse && branchlength < length)) length = branchlength; - if (op != OP_ALT) return length; + if (op != OP_ALT || length == 0) return length; + nextbranch = cc + GET(cc, 1); cc += 1 + LINK_SIZE; branchlength = 0; had_recurse = FALSE; @@ -238,6 +248,8 @@ for (;;) case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERT_NA: + case OP_ASSERTBACK_NA: do cc += GET(cc, 1); while (*cc == OP_ALT); /* Fall through */ @@ -451,15 +463,17 @@ for (;;) If PCRE2_MATCH_UNSET_BACKREF is set, a backreference to an unset bracket matches an empty string (by default it causes a matching failure), so in - that case we must set the minimum length to zero. */ + that case we must set the minimum length to zero. - /* Duplicate named pattern back reference. We cannot reliably find a length - for this if duplicate numbers are present in the pattern. */ + For backreferenes, if duplicate numbers are present in the pattern we check + for a reference to a duplicate. If it is, we don't know which version will + be referenced, so we have to set the minimum length to zero. */ + + /* Duplicate named pattern back reference. */ case OP_DNREF: case OP_DNREFI: - if (dupcapused) return -1; - if ((re->overall_options & PCRE2_MATCH_UNSET_BACKREF) == 0) + if (!dupcapused && (re->overall_options & PCRE2_MATCH_UNSET_BACKREF) == 0) { int count = GET2(cc, 1+IMM2_SIZE); PCRE2_UCHAR *slot = @@ -482,28 +496,32 @@ for (;;) ce = cs = (PCRE2_UCHAR *)PRIV(find_bracket)(startcode, utf, recno); if (cs == NULL) return -2; do ce += GET(ce, 1); while (*ce == OP_ALT); - if (cc > cs && cc < ce) /* Simple recursion */ - { - dd = 0; - had_recurse = TRUE; - } - else + + dd = 0; + if (!dupcapused || + (PCRE2_UCHAR *)PRIV(find_bracket)(ce, utf, recno) == NULL) { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) - if (r->group == cs) break; - if (r != NULL) /* Mutual recursion */ + if (cc > cs && cc < ce) /* Simple recursion */ { - dd = 0; had_recurse = TRUE; } else { - this_recurse.prev = recurses; - this_recurse.group = cs; - dd = find_minlength(re, cs, startcode, utf, &this_recurse, - countptr, backref_cache); - if (dd < 0) return dd; + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) + if (r->group == cs) break; + if (r != NULL) /* Mutual recursion */ + { + had_recurse = TRUE; + } + else + { + this_recurse.prev = recurses; /* No recursion */ + this_recurse.group = cs; + dd = find_minlength(re, cs, startcode, utf, &this_recurse, + countptr, backref_cache); + if (dd < 0) return dd; + } } } @@ -521,48 +539,51 @@ for (;;) cc += 1 + 2*IMM2_SIZE; goto REPEAT_BACK_REFERENCE; - /* Single back reference. We cannot find a length for this if duplicate - numbers are present in the pattern. */ + /* Single back reference by number. References by name are converted to by + number when there is no duplication. */ case OP_REF: case OP_REFI: - if (dupcapused) return -1; recno = GET2(cc, 1); if (recno <= backref_cache[0] && backref_cache[recno] >= 0) d = backref_cache[recno]; else { int i; + d = 0; + if ((re->overall_options & PCRE2_MATCH_UNSET_BACKREF) == 0) { ce = cs = (PCRE2_UCHAR *)PRIV(find_bracket)(startcode, utf, recno); if (cs == NULL) return -2; do ce += GET(ce, 1); while (*ce == OP_ALT); - if (cc > cs && cc < ce) /* Simple recursion */ - { - d = 0; - had_recurse = TRUE; - } - else + + if (!dupcapused || + (PCRE2_UCHAR *)PRIV(find_bracket)(ce, utf, recno) == NULL) { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; - if (r != NULL) /* Mutual recursion */ + if (cc > cs && cc < ce) /* Simple recursion */ { - d = 0; had_recurse = TRUE; } else { - this_recurse.prev = recurses; - this_recurse.group = cs; - d = find_minlength(re, cs, startcode, utf, &this_recurse, countptr, - backref_cache); - if (d < 0) return d; + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; + if (r != NULL) /* Mutual recursion */ + { + had_recurse = TRUE; + } + else /* No recursion */ + { + this_recurse.prev = recurses; + this_recurse.group = cs; + d = find_minlength(re, cs, startcode, utf, &this_recurse, countptr, + backref_cache); + if (d < 0) return d; + } } } } - else d = 0; backref_cache[recno] = d; for (i = backref_cache[0] + 1; i < recno; i++) backref_cache[i] = -1; @@ -751,15 +772,19 @@ Arguments: p points to the first code unit of the character caseless TRUE if caseless utf TRUE for UTF mode + ucp TRUE for UCP mode Returns: pointer after the character */ static PCRE2_SPTR -set_table_bit(pcre2_real_code *re, PCRE2_SPTR p, BOOL caseless, BOOL utf) +set_table_bit(pcre2_real_code *re, PCRE2_SPTR p, BOOL caseless, BOOL utf, + BOOL ucp) { uint32_t c = *p++; /* First code unit */ -(void)utf; /* Stop compiler warning when UTF not supported */ + +(void)utf; /* Stop compiler warnings when UTF not supported */ +(void)ucp; /* In 16-bit and 32-bit modes, code units greater than 0xff set the bit for 0xff. */ @@ -789,22 +814,26 @@ if (utf) if (caseless) { #ifdef SUPPORT_UNICODE - if (utf) + if (utf || ucp) { -#if PCRE2_CODE_UNIT_WIDTH == 8 - PCRE2_UCHAR buff[6]; c = UCD_OTHERCASE(c); - (void)PRIV(ord2utf)(c, buff); - SET_BIT(buff[0]); +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (utf) + { + PCRE2_UCHAR buff[6]; + (void)PRIV(ord2utf)(c, buff); + SET_BIT(buff[0]); + } + else if (c < 256) SET_BIT(c); #else /* 16-bit or 32-bit mode */ - c = UCD_OTHERCASE(c); if (c > 0xff) SET_BIT(0xff); else SET_BIT(c); #endif } + else #endif /* SUPPORT_UNICODE */ - /* Not UTF */ + /* Not UTF or UCP */ if (MAX_255(c)) SET_BIT(re->tables[fcc_offset + c]); } @@ -888,7 +917,7 @@ if (table_limit != 32) for (c = 24; c < 32; c++) re->start_bitmap[c] = 0xff; /************************************************* -* Create bitmap of starting bytes * +* Create bitmap of starting code units * *************************************************/ /* This function scans a compiled unanchored expression recursively and @@ -903,19 +932,26 @@ The SSB_CONTINUE return is useful for parenthesized groups in patterns such as must continue at the outer level to find at least one mandatory code unit. At the outermost level, this function fails unless the result is SSB_DONE. +We restrict recursion (for nested groups) to 1000 to avoid stack overflow +issues. + Arguments: re points to the compiled regex block code points to an expression utf TRUE if in UTF mode + ucp TRUE if in UCP mode + depthptr pointer to recurse depth Returns: SSB_FAIL => Failed to find any starting code units SSB_DONE => Found mandatory starting code units SSB_CONTINUE => Found optional starting code units SSB_UNKNOWN => Hit an unrecognized opcode + SSB_TOODEEP => Recursion is too deep */ static int -set_start_bits(pcre2_real_code *re, PCRE2_SPTR code, BOOL utf) +set_start_bits(pcre2_real_code *re, PCRE2_SPTR code, BOOL utf, BOOL ucp, + int *depthptr) { uint32_t c; int yield = SSB_DONE; @@ -926,6 +962,9 @@ int table_limit = utf? 16:32; int table_limit = 32; #endif +*depthptr += 1; +if (*depthptr > 1000) return SSB_TOODEEP; + do { BOOL try_next = TRUE; @@ -938,6 +977,9 @@ do { int rc; uint8_t *classmap = NULL; +#ifdef SUPPORT_WIDE_CHARS + PCRE2_UCHAR xclassflags; +#endif switch(*tcode) { @@ -1078,13 +1120,18 @@ do case OP_ONCE: case OP_SCRIPT_RUN: case OP_ASSERT: - rc = set_start_bits(re, tcode, utf); - if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; - if (rc == SSB_DONE) try_next = FALSE; else + case OP_ASSERT_NA: + rc = set_start_bits(re, tcode, utf, ucp, depthptr); + if (rc == SSB_DONE) + { + try_next = FALSE; + } + else if (rc == SSB_CONTINUE) { do tcode += GET(tcode, 1); while (*tcode == OP_ALT); tcode += 1 + LINK_SIZE; } + else return rc; /* FAIL, UNKNOWN, or TOODEEP */ break; /* If we hit ALT or KET, it means we haven't found anything mandatory in @@ -1120,6 +1167,7 @@ do case OP_ASSERT_NOT: case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: + case OP_ASSERTBACK_NA: do tcode += GET(tcode, 1); while (*tcode == OP_ALT); tcode += 1 + LINK_SIZE; break; @@ -1129,8 +1177,8 @@ do case OP_BRAZERO: case OP_BRAMINZERO: case OP_BRAPOSZERO: - rc = set_start_bits(re, ++tcode, utf); - if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; + rc = set_start_bits(re, ++tcode, utf, ucp, depthptr); + if (rc == SSB_FAIL || rc == SSB_UNKNOWN || rc == SSB_TOODEEP) return rc; do tcode += GET(tcode,1); while (*tcode == OP_ALT); tcode += 1 + LINK_SIZE; break; @@ -1151,7 +1199,7 @@ do case OP_QUERY: case OP_MINQUERY: case OP_POSQUERY: - tcode = set_table_bit(re, tcode + 1, FALSE, utf); + tcode = set_table_bit(re, tcode + 1, FALSE, utf, ucp); break; case OP_STARI: @@ -1160,7 +1208,7 @@ do case OP_QUERYI: case OP_MINQUERYI: case OP_POSQUERYI: - tcode = set_table_bit(re, tcode + 1, TRUE, utf); + tcode = set_table_bit(re, tcode + 1, TRUE, utf, ucp); break; /* Single-char upto sets the bit and tries the next */ @@ -1168,13 +1216,13 @@ do case OP_UPTO: case OP_MINUPTO: case OP_POSUPTO: - tcode = set_table_bit(re, tcode + 1 + IMM2_SIZE, FALSE, utf); + tcode = set_table_bit(re, tcode + 1 + IMM2_SIZE, FALSE, utf, ucp); break; case OP_UPTOI: case OP_MINUPTOI: case OP_POSUPTOI: - tcode = set_table_bit(re, tcode + 1 + IMM2_SIZE, TRUE, utf); + tcode = set_table_bit(re, tcode + 1 + IMM2_SIZE, TRUE, utf, ucp); break; /* At least one single char sets the bit and stops */ @@ -1186,7 +1234,7 @@ do case OP_PLUS: case OP_MINPLUS: case OP_POSPLUS: - (void)set_table_bit(re, tcode + 1, FALSE, utf); + (void)set_table_bit(re, tcode + 1, FALSE, utf, ucp); try_next = FALSE; break; @@ -1197,7 +1245,7 @@ do case OP_PLUSI: case OP_MINPLUSI: case OP_POSPLUSI: - (void)set_table_bit(re, tcode + 1, TRUE, utf); + (void)set_table_bit(re, tcode + 1, TRUE, utf, ucp); try_next = FALSE; break; @@ -1444,20 +1492,59 @@ do negative XCLASS without a map, give up. If there are no property checks, there must be wide characters on the XCLASS list, because otherwise an XCLASS would not have been created. This means that code points >= 255 - are always potential starters. */ + are potential starters. In the UTF-8 case we can scan them and set bits + for the relevant leading bytes. */ #ifdef SUPPORT_WIDE_CHARS case OP_XCLASS: - if ((tcode[1 + LINK_SIZE] & XCL_HASPROP) != 0 || - (tcode[1 + LINK_SIZE] & (XCL_MAP|XCL_NOT)) == XCL_NOT) + xclassflags = tcode[1 + LINK_SIZE]; + if ((xclassflags & XCL_HASPROP) != 0 || + (xclassflags & (XCL_MAP|XCL_NOT)) == XCL_NOT) return SSB_FAIL; /* We have a positive XCLASS or a negative one without a map. Set up the map pointer if there is one, and fall through. */ - classmap = ((tcode[1 + LINK_SIZE] & XCL_MAP) == 0)? NULL : + classmap = ((xclassflags & XCL_MAP) == 0)? NULL : (uint8_t *)(tcode + 1 + LINK_SIZE + 1); -#endif + + /* In UTF-8 mode, scan the character list and set bits for leading bytes, + then jump to handle the map. */ + +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (utf && (xclassflags & XCL_NOT) == 0) + { + PCRE2_UCHAR b, e; + PCRE2_SPTR p = tcode + 1 + LINK_SIZE + 1 + ((classmap == NULL)? 0:32); + tcode += GET(tcode, 1); + + for (;;) switch (*p++) + { + case XCL_SINGLE: + b = *p++; + while ((*p & 0xc0) == 0x80) p++; + re->start_bitmap[b/8] |= (1u << (b&7)); + break; + + case XCL_RANGE: + b = *p++; + while ((*p & 0xc0) == 0x80) p++; + e = *p++; + while ((*p & 0xc0) == 0x80) p++; + for (; b <= e; b++) + re->start_bitmap[b/8] |= (1u << (b&7)); + break; + + case XCL_END: + goto HANDLE_CLASSMAP; + + default: + return SSB_UNKNOWN; /* Internal error, should not occur */ + } + } +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 */ +#endif /* SUPPORT_WIDE_CHARS */ + /* It seems that the fall through comment must be outside the #ifdef if it is to avoid the gcc compiler warning. */ @@ -1499,6 +1586,9 @@ do greater than 127. In fact, there are only two possible starting bytes for characters in the range 128 - 255. */ +#if defined SUPPORT_WIDE_CHARS && PCRE2_CODE_UNIT_WIDTH == 8 + HANDLE_CLASSMAP: +#endif if (classmap != NULL) { #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 @@ -1569,7 +1659,9 @@ return yield; /* This function is handed a compiled expression that it must study to produce information that will speed up the matching. -Argument: points to the compiled expression +Argument: + re points to the compiled expression + Returns: 0 normally; non-zero should never normally occur 1 unknown opcode in set_start_bits 2 missing capturing bracket @@ -1579,10 +1671,10 @@ Returns: 0 normally; non-zero should never normally occur int PRIV(study)(pcre2_real_code *re) { -int min; int count = 0; PCRE2_UCHAR *code; BOOL utf = (re->overall_options & PCRE2_UTF) != 0; +BOOL ucp = (re->overall_options & PCRE2_UCP) != 0; /* Find start of compiled code */ @@ -1595,27 +1687,124 @@ code units. */ if ((re->flags & (PCRE2_FIRSTSET|PCRE2_STARTLINE)) == 0) { - int rc = set_start_bits(re, code, utf); + int depth = 0; + int rc = set_start_bits(re, code, utf, ucp, &depth); if (rc == SSB_UNKNOWN) return 1; - if (rc == SSB_DONE) re->flags |= PCRE2_FIRSTMAPSET; + + /* If a list of starting code units was set up, scan the list to see if only + one or two were listed. Having only one listed is rare because usually a + single starting code unit will have been recognized and PCRE2_FIRSTSET set. + If two are listed, see if they are caseless versions of the same character; + if so we can replace the list with a caseless first code unit. This gives + better performance and is plausibly worth doing for patterns such as [Ww]ord + or (word|WORD). */ + + if (rc == SSB_DONE) + { + int i; + int a = -1; + int b = -1; + uint8_t *p = re->start_bitmap; + uint32_t flags = PCRE2_FIRSTMAPSET; + + for (i = 0; i < 256; p++, i += 8) + { + uint8_t x = *p; + if (x != 0) + { + int c; + uint8_t y = x & (~x + 1); /* Least significant bit */ + if (y != x) goto DONE; /* More than one bit set */ + + /* In the 16-bit and 32-bit libraries, the bit for 0xff means "0xff and + all wide characters", so we cannot use it here. */ + +#if PCRE2_CODE_UNIT_WIDTH != 8 + if (i == 248 && x == 0x80) goto DONE; +#endif + + /* Compute the character value */ + + c = i; + switch (x) + { + case 1: break; + case 2: c += 1; break; case 4: c += 2; break; + case 8: c += 3; break; case 16: c += 4; break; + case 32: c += 5; break; case 64: c += 6; break; + case 128: c += 7; break; + } + + /* c contains the code unit value, in the range 0-255. In 8-bit UTF + mode, only values < 128 can be used. In all the other cases, c is a + character value. */ + +#if PCRE2_CODE_UNIT_WIDTH == 8 + if (utf && c > 127) goto DONE; +#endif + if (a < 0) a = c; /* First one found, save in a */ + else if (b < 0) /* Second one found */ + { + int d = TABLE_GET((unsigned int)c, re->tables + fcc_offset, c); + +#ifdef SUPPORT_UNICODE + if (utf || ucp) + { + if (UCD_CASESET(c) != 0) goto DONE; /* Multiple case set */ + if (c > 127) d = UCD_OTHERCASE(c); + } +#endif /* SUPPORT_UNICODE */ + + if (d != a) goto DONE; /* Not the other case of a */ + b = c; /* Save second in b */ + } + else goto DONE; /* More than two characters found */ + } + } + + /* Replace the start code unit bits with a first code unit, but only if it + is not the same as a required later code unit. This is because a search for + a required code unit starts after an explicit first code unit, but at a + code unit found from the bitmap. Patterns such as /a*a/ don't work + if both the start unit and required unit are the same. */ + + if (a >= 0 && + ( + (re->flags & PCRE2_LASTSET) == 0 || + ( + re->last_codeunit != (uint32_t)a && + (b < 0 || re->last_codeunit != (uint32_t)b) + ) + )) + { + re->first_codeunit = a; + flags = PCRE2_FIRSTSET; + if (b >= 0) flags |= PCRE2_FIRSTCASELESS; + } + + DONE: + re->flags |= flags; + } } /* Find the minimum length of subject string. If the pattern can match an empty -string, the minimum length is already known. If there are more back references -than the size of the vector we are going to cache them in, do nothing. A -pattern that complicated will probably take a long time to analyze and may in -any case turn out to be too complicated. Note that back reference minima are -held as 16-bit numbers. */ - -if ((re->flags & PCRE2_MATCH_EMPTY) == 0 && +string, the minimum length is already known. If the pattern contains (*ACCEPT) +all bets are off, and we don't even try to find a minimum length. If there are +more back references than the size of the vector we are going to cache them in, +do nothing. A pattern that complicated will probably take a long time to +analyze and may in any case turn out to be too complicated. Note that back +reference minima are held as 16-bit numbers. */ + +if ((re->flags & (PCRE2_MATCH_EMPTY|PCRE2_HASACCEPT)) == 0 && re->top_backref <= MAX_CACHE_BACKREF) { + int min; int backref_cache[MAX_CACHE_BACKREF+1]; backref_cache[0] = 0; /* Highest one that is set */ min = find_minlength(re, code, code, utf, NULL, &count, backref_cache); switch(min) { - case -1: /* \C in UTF mode or (*ACCEPT) or over-complex regex */ + case -1: /* \C in UTF mode or over-complex regex */ break; /* Leave minlength unchanged (will be zero) */ case -2: @@ -1625,8 +1814,7 @@ if ((re->flags & PCRE2_MATCH_EMPTY) == 0 && return 3; /* unrecognized opcode */ default: - if (min > UINT16_MAX) min = UINT16_MAX; - re->minlength = min; + re->minlength = (min > UINT16_MAX)? UINT16_MAX : min; break; } } diff --git a/dist2/src/pcre2_substitute.c b/dist2/src/pcre2_substitute.c index ec3dd66d..981a106a 100644 --- a/dist2/src/pcre2_substitute.c +++ b/dist2/src/pcre2_substitute.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2019 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -49,8 +49,9 @@ POSSIBILITY OF SUCH DAMAGE. #define SUBSTITUTE_OPTIONS \ (PCRE2_SUBSTITUTE_EXTENDED|PCRE2_SUBSTITUTE_GLOBAL| \ - PCRE2_SUBSTITUTE_OVERFLOW_LENGTH|PCRE2_SUBSTITUTE_UNKNOWN_UNSET| \ - PCRE2_SUBSTITUTE_UNSET_EMPTY) + PCRE2_SUBSTITUTE_LITERAL|PCRE2_SUBSTITUTE_MATCHED| \ + PCRE2_SUBSTITUTE_OVERFLOW_LENGTH|PCRE2_SUBSTITUTE_REPLACEMENT_ONLY| \ + PCRE2_SUBSTITUTE_UNKNOWN_UNSET|PCRE2_SUBSTITUTE_UNSET_EMPTY) @@ -194,6 +195,7 @@ overflow, either give an error immediately, or keep on, accumulating the length. */ #define CHECKMEMCPY(from,length) \ + { \ if (!overflowed && lengthleft < length) \ { \ if ((suboptions & PCRE2_SUBSTITUTE_OVERFLOW_LENGTH) == 0) goto NOROOM; \ @@ -209,7 +211,8 @@ length. */ memcpy(buffer + buff_offset, from, CU2BYTES(length)); \ buff_offset += length; \ lengthleft -= length; \ - } + } \ + } /* Here's the function */ @@ -226,11 +229,14 @@ int forcecasereset = 0; uint32_t ovector_count; uint32_t goptions = 0; uint32_t suboptions; -BOOL match_data_created = FALSE; -BOOL literal = FALSE; +pcre2_match_data *internal_match_data = NULL; +BOOL escaped_literal = FALSE; BOOL overflowed = FALSE; +BOOL use_existing_match; +BOOL replacement_only; #ifdef SUPPORT_UNICODE BOOL utf = (code->overall_options & PCRE2_UTF) != 0; +BOOL ucp = (code->overall_options & PCRE2_UCP) != 0; #endif PCRE2_UCHAR temp[6]; PCRE2_SPTR ptr; @@ -248,23 +254,54 @@ lengthleft = buff_length = *blength; *blength = PCRE2_UNSET; ovecsave[0] = ovecsave[1] = ovecsave[2] = PCRE2_UNSET; -/* Partial matching is not valid. This must come after setting *blength to +/* Partial matching is not valid. This must come after setting *blength to PCRE2_UNSET, so as not to imply an offset in the replacement. */ if ((options & (PCRE2_PARTIAL_HARD|PCRE2_PARTIAL_SOFT)) != 0) return PCRE2_ERROR_BADOPTION; -/* If no match data block is provided, create one. */ +/* Check for using a match that has already happened. Note that the subject +pointer in the match data may be NULL after a no-match. */ + +use_existing_match = ((options & PCRE2_SUBSTITUTE_MATCHED) != 0); +replacement_only = ((options & PCRE2_SUBSTITUTE_REPLACEMENT_ONLY) != 0); + +/* If starting from an existing match, there must be an externally provided +match data block. We create an internal match_data block in two cases: (a) an +external one is not supplied (and we are not starting from an existing match); +(b) an existing match is to be used for the first substitution. In the latter +case, we copy the existing match into the internal block. This ensures that no +changes are made to the existing match data block. */ if (match_data == NULL) { + pcre2_general_context *gcontext; + if (use_existing_match) return PCRE2_ERROR_NULL; + gcontext = (mcontext == NULL)? + (pcre2_general_context *)code : + (pcre2_general_context *)mcontext; + match_data = internal_match_data = + pcre2_match_data_create_from_pattern(code, gcontext); + if (internal_match_data == NULL) return PCRE2_ERROR_NOMEMORY; + } + +else if (use_existing_match) + { pcre2_general_context *gcontext = (mcontext == NULL)? (pcre2_general_context *)code : (pcre2_general_context *)mcontext; - match_data = pcre2_match_data_create_from_pattern(code, gcontext); - if (match_data == NULL) return PCRE2_ERROR_NOMEMORY; - match_data_created = TRUE; + int pairs = (code->top_bracket + 1 < match_data->oveccount)? + code->top_bracket + 1 : match_data->oveccount; + internal_match_data = pcre2_match_data_create(match_data->oveccount, + gcontext); + if (internal_match_data == NULL) return PCRE2_ERROR_NOMEMORY; + memcpy(internal_match_data, match_data, offsetof(pcre2_match_data, ovector) + + 2*pairs*sizeof(PCRE2_SIZE)); + match_data = internal_match_data; } + +/* Remember ovector details */ + ovector = pcre2_get_ovector_pointer(match_data); ovector_count = pcre2_get_ovector_count(match_data); @@ -286,7 +323,7 @@ repend = replacement + rlength; #ifdef SUPPORT_UNICODE if (utf && (options & PCRE2_NO_UTF_CHECK) == 0) { - rc = PRIV(valid_utf)(replacement, rlength, &(match_data->rightchar)); + rc = PRIV(valid_utf)(replacement, rlength, &(match_data->startchar)); if (rc != 0) { match_data->leftchar = 0; @@ -300,7 +337,7 @@ if (utf && (options & PCRE2_NO_UTF_CHECK) == 0) suboptions = options & SUBSTITUTE_OPTIONS; options &= ~SUBSTITUTE_OPTIONS; -/* Copy up to the start offset */ +/* Error if the start match offset is greater than the length of the subject. */ if (start_offset > length) { @@ -308,9 +345,13 @@ if (start_offset > length) rc = PCRE2_ERROR_BADOFFSET; goto EXIT; } -CHECKMEMCPY(subject, start_offset); -/* Loop for global substituting. */ +/* Copy up to the start offset, unless only the replacement is required. */ + +if (!replacement_only) CHECKMEMCPY(subject, start_offset); + +/* Loop for global substituting. If PCRE2_SUBSTITUTE_MATCHED is set, the first +match is taken from the match_data that was passed in. */ subs = 0; do @@ -318,7 +359,12 @@ do PCRE2_SPTR ptrstack[PTR_STACK_SIZE]; uint32_t ptrstackptr = 0; - rc = pcre2_match(code, subject, length, start_offset, options|goptions, + if (use_existing_match) + { + rc = match_data->rc; + use_existing_match = FALSE; + } + else rc = pcre2_match(code, subject, length, start_offset, options|goptions, match_data, mcontext); #ifdef SUPPORT_UNICODE @@ -364,44 +410,44 @@ do #endif } - /* Copy what we have advanced past, reset the special global options, and - continue to the next match. */ + /* Copy what we have advanced past (unless not required), reset the special + global options, and continue to the next match. */ fraglength = start_offset - save_start; - CHECKMEMCPY(subject + save_start, fraglength); + if (!replacement_only) CHECKMEMCPY(subject + save_start, fraglength); goptions = 0; continue; } /* Handle a successful match. Matches that use \K to end before they start or start before the current point in the subject are not supported. */ - + if (ovector[1] < ovector[0] || ovector[0] < start_offset) { rc = PCRE2_ERROR_BADSUBSPATTERN; goto EXIT; } - - /* Check for the same match as previous. This is legitimate after matching an + + /* Check for the same match as previous. This is legitimate after matching an empty string that starts after the initial match offset. We have tried again at the match point in case the pattern is one like /(?<=\G.)/ which can never match at its starting point, so running the match achieves the bumpalong. If we do get the same (null) match at the original match point, it isn't such a pattern, so we now do the empty string magic. In all other cases, a repeat match should never occur. */ - + if (ovecsave[0] == ovector[0] && ovecsave[1] == ovector[1]) - { - if (ovector[0] == ovector[1] && ovecsave[2] != start_offset) - { - goptions = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; - ovecsave[2] = start_offset; - continue; /* Back to the top of the loop */ + { + if (ovector[0] == ovector[1] && ovecsave[2] != start_offset) + { + goptions = PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED; + ovecsave[2] = start_offset; + continue; /* Back to the top of the loop */ } rc = PCRE2_ERROR_INTERNAL_DUPMATCH; - goto EXIT; - } - + goto EXIT; + } + /* Count substitutions with a paranoid check for integer overflow; surely no real call to this function would ever hit this! */ @@ -412,21 +458,30 @@ do } subs++; - /* Copy the text leading up to the match, and remember where the insert - begins and how many ovector pairs are set. */ + /* Copy the text leading up to the match (unless not required), and remember + where the insert begins and how many ovector pairs are set. */ if (rc == 0) rc = ovector_count; fraglength = ovector[0] - start_offset; - CHECKMEMCPY(subject + start_offset, fraglength); + if (!replacement_only) CHECKMEMCPY(subject + start_offset, fraglength); scb.output_offsets[0] = buff_offset; scb.oveccount = rc; - /* Process the replacement string. Literal mode is set by \Q, but only in - extended mode when backslashes are being interpreted. In extended mode we - must handle nested substrings that are to be reprocessed. */ + /* Process the replacement string. If the entire replacement is literal, just + copy it with length check. */ ptr = replacement; - for (;;) + if ((suboptions & PCRE2_SUBSTITUTE_LITERAL) != 0) + { + CHECKMEMCPY(ptr, rlength); + } + + /* Within a non-literal replacement, which must be scanned character by + character, local literal mode can be set by \Q, but only in extended mode + when backslashes are being interpreted. In extended mode we must handle + nested substrings that are to be reprocessed. */ + + else for (;;) { uint32_t ch; unsigned int chlen; @@ -443,11 +498,11 @@ do /* Handle the next character */ - if (literal) + if (escaped_literal) { if (ptr[0] == CHAR_BACKSLASH && ptr < repend - 1 && ptr[1] == CHAR_E) { - literal = FALSE; + escaped_literal = FALSE; ptr += 2; continue; } @@ -704,7 +759,7 @@ do if (forcecase != 0) { #ifdef SUPPORT_UNICODE - if (utf) + if (utf || ucp) { uint32_t type = UCD_CHARTYPE(ch); if (PRIV(ucp_gentype)[type] == ucp_L && @@ -784,7 +839,7 @@ do continue; case ESC_Q: - literal = TRUE; + escaped_literal = TRUE; continue; case 0: /* Data character */ @@ -806,7 +861,7 @@ do if (forcecase != 0) { #ifdef SUPPORT_UNICODE - if (utf) + if (utf || ucp) { uint32_t type = UCD_CHARTYPE(ch); if (PRIV(ucp_gentype)[type] == ucp_L && @@ -835,53 +890,59 @@ do } /* End handling a literal code unit */ } /* End of loop for scanning the replacement. */ - /* The replacement has been copied to the output, or its size has been - remembered. Do the callout if there is one and we have done an actual + /* The replacement has been copied to the output, or its size has been + remembered. Do the callout if there is one and we have done an actual replacement. */ - + if (!overflowed && mcontext != NULL && mcontext->substitute_callout != NULL) { - scb.subscount = subs; + scb.subscount = subs; scb.output_offsets[1] = buff_offset; - rc = mcontext->substitute_callout(&scb, mcontext->substitute_callout_data); + rc = mcontext->substitute_callout(&scb, mcontext->substitute_callout_data); - /* A non-zero return means cancel this substitution. Instead, copy the + /* A non-zero return means cancel this substitution. Instead, copy the matched string fragment. */ if (rc != 0) { PCRE2_SIZE newlength = scb.output_offsets[1] - scb.output_offsets[0]; PCRE2_SIZE oldlength = ovector[1] - ovector[0]; - + buff_offset -= newlength; lengthleft += newlength; - CHECKMEMCPY(subject + ovector[0], oldlength); - + if (!replacement_only) CHECKMEMCPY(subject + ovector[0], oldlength); + /* A negative return means do not do any more. */ - + if (rc < 0) suboptions &= (~PCRE2_SUBSTITUTE_GLOBAL); } - } - + } + /* Save the details of this match. See above for how this data is used. If we - matched an empty string, do the magic for global matches. Finally, update the - start offset to point to the rest of the subject string. */ - - ovecsave[0] = ovector[0]; - ovecsave[1] = ovector[1]; + matched an empty string, do the magic for global matches. Update the start + offset to point to the rest of the subject string. If we re-used an existing + match for the first match, switch to the internal match data block. */ + + ovecsave[0] = ovector[0]; + ovecsave[1] = ovector[1]; ovecsave[2] = start_offset; - + goptions = (ovector[0] != ovector[1] || ovector[0] > start_offset)? 0 : PCRE2_ANCHORED|PCRE2_NOTEMPTY_ATSTART; start_offset = ovector[1]; } while ((suboptions & PCRE2_SUBSTITUTE_GLOBAL) != 0); /* Repeat "do" loop */ -/* Copy the rest of the subject. */ +/* Copy the rest of the subject unless not required, and terminate the output +with a binary zero. */ + +if (!replacement_only) + { + fraglength = length - start_offset; + CHECKMEMCPY(subject + start_offset, fraglength); + } -fraglength = length - start_offset; -CHECKMEMCPY(subject + start_offset, fraglength); temp[0] = 0; -CHECKMEMCPY(temp , 1); +CHECKMEMCPY(temp, 1); /* If overflowed is set it means the PCRE2_SUBSTITUTE_OVERFLOW_LENGTH is set, and matching has carried on after a full buffer, in order to compute the length @@ -903,7 +964,7 @@ else } EXIT: -if (match_data_created) pcre2_match_data_free(match_data); +if (internal_match_data != NULL) pcre2_match_data_free(internal_match_data); else match_data->rc = rc; return rc; diff --git a/dist2/src/pcre2_tables.c b/dist2/src/pcre2_tables.c index 84019361..b10de45e 100644 --- a/dist2/src/pcre2_tables.c +++ b/dist2/src/pcre2_tables.c @@ -265,6 +265,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Chakma0 STR_C STR_h STR_a STR_k STR_m STR_a "\0" #define STRING_Cham0 STR_C STR_h STR_a STR_m "\0" #define STRING_Cherokee0 STR_C STR_h STR_e STR_r STR_o STR_k STR_e STR_e "\0" +#define STRING_Chorasmian0 STR_C STR_h STR_o STR_r STR_a STR_s STR_m STR_i STR_a STR_n "\0" #define STRING_Cn0 STR_C STR_n "\0" #define STRING_Co0 STR_C STR_o "\0" #define STRING_Common0 STR_C STR_o STR_m STR_m STR_o STR_n "\0" @@ -275,10 +276,12 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Cyrillic0 STR_C STR_y STR_r STR_i STR_l STR_l STR_i STR_c "\0" #define STRING_Deseret0 STR_D STR_e STR_s STR_e STR_r STR_e STR_t "\0" #define STRING_Devanagari0 STR_D STR_e STR_v STR_a STR_n STR_a STR_g STR_a STR_r STR_i "\0" +#define STRING_Dives_Akuru0 STR_D STR_i STR_v STR_e STR_s STR_UNDERSCORE STR_A STR_k STR_u STR_r STR_u "\0" #define STRING_Dogra0 STR_D STR_o STR_g STR_r STR_a "\0" #define STRING_Duployan0 STR_D STR_u STR_p STR_l STR_o STR_y STR_a STR_n "\0" #define STRING_Egyptian_Hieroglyphs0 STR_E STR_g STR_y STR_p STR_t STR_i STR_a STR_n STR_UNDERSCORE STR_H STR_i STR_e STR_r STR_o STR_g STR_l STR_y STR_p STR_h STR_s "\0" #define STRING_Elbasan0 STR_E STR_l STR_b STR_a STR_s STR_a STR_n "\0" +#define STRING_Elymaic0 STR_E STR_l STR_y STR_m STR_a STR_i STR_c "\0" #define STRING_Ethiopic0 STR_E STR_t STR_h STR_i STR_o STR_p STR_i STR_c "\0" #define STRING_Georgian0 STR_G STR_e STR_o STR_r STR_g STR_i STR_a STR_n "\0" #define STRING_Glagolitic0 STR_G STR_l STR_a STR_g STR_o STR_l STR_i STR_t STR_i STR_c "\0" @@ -305,6 +308,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Katakana0 STR_K STR_a STR_t STR_a STR_k STR_a STR_n STR_a "\0" #define STRING_Kayah_Li0 STR_K STR_a STR_y STR_a STR_h STR_UNDERSCORE STR_L STR_i "\0" #define STRING_Kharoshthi0 STR_K STR_h STR_a STR_r STR_o STR_s STR_h STR_t STR_h STR_i "\0" +#define STRING_Khitan_Small_Script0 STR_K STR_h STR_i STR_t STR_a STR_n STR_UNDERSCORE STR_S STR_m STR_a STR_l STR_l STR_UNDERSCORE STR_S STR_c STR_r STR_i STR_p STR_t "\0" #define STRING_Khmer0 STR_K STR_h STR_m STR_e STR_r "\0" #define STRING_Khojki0 STR_K STR_h STR_o STR_j STR_k STR_i "\0" #define STRING_Khudawadi0 STR_K STR_h STR_u STR_d STR_a STR_w STR_a STR_d STR_i "\0" @@ -348,6 +352,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Myanmar0 STR_M STR_y STR_a STR_n STR_m STR_a STR_r "\0" #define STRING_N0 STR_N "\0" #define STRING_Nabataean0 STR_N STR_a STR_b STR_a STR_t STR_a STR_e STR_a STR_n "\0" +#define STRING_Nandinagari0 STR_N STR_a STR_n STR_d STR_i STR_n STR_a STR_g STR_a STR_r STR_i "\0" #define STRING_Nd0 STR_N STR_d "\0" #define STRING_New_Tai_Lue0 STR_N STR_e STR_w STR_UNDERSCORE STR_T STR_a STR_i STR_UNDERSCORE STR_L STR_u STR_e "\0" #define STRING_Newa0 STR_N STR_e STR_w STR_a "\0" @@ -355,6 +360,7 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Nl0 STR_N STR_l "\0" #define STRING_No0 STR_N STR_o "\0" #define STRING_Nushu0 STR_N STR_u STR_s STR_h STR_u "\0" +#define STRING_Nyiakeng_Puachue_Hmong0 STR_N STR_y STR_i STR_a STR_k STR_e STR_n STR_g STR_UNDERSCORE STR_P STR_u STR_a STR_c STR_h STR_u STR_e STR_UNDERSCORE STR_H STR_m STR_o STR_n STR_g "\0" #define STRING_Ogham0 STR_O STR_g STR_h STR_a STR_m "\0" #define STRING_Ol_Chiki0 STR_O STR_l STR_UNDERSCORE STR_C STR_h STR_i STR_k STR_i "\0" #define STRING_Old_Hungarian0 STR_O STR_l STR_d STR_UNDERSCORE STR_H STR_u STR_n STR_g STR_a STR_r STR_i STR_a STR_n "\0" @@ -419,12 +425,14 @@ strings to make sure that UTF-8 support works on EBCDIC platforms. */ #define STRING_Ugaritic0 STR_U STR_g STR_a STR_r STR_i STR_t STR_i STR_c "\0" #define STRING_Unknown0 STR_U STR_n STR_k STR_n STR_o STR_w STR_n "\0" #define STRING_Vai0 STR_V STR_a STR_i "\0" +#define STRING_Wancho0 STR_W STR_a STR_n STR_c STR_h STR_o "\0" #define STRING_Warang_Citi0 STR_W STR_a STR_r STR_a STR_n STR_g STR_UNDERSCORE STR_C STR_i STR_t STR_i "\0" #define STRING_Xan0 STR_X STR_a STR_n "\0" #define STRING_Xps0 STR_X STR_p STR_s "\0" #define STRING_Xsp0 STR_X STR_s STR_p "\0" #define STRING_Xuc0 STR_X STR_u STR_c "\0" #define STRING_Xwd0 STR_X STR_w STR_d "\0" +#define STRING_Yezidi0 STR_Y STR_e STR_z STR_i STR_d STR_i "\0" #define STRING_Yi0 STR_Y STR_i "\0" #define STRING_Z0 STR_Z "\0" #define STRING_Zanabazar_Square0 STR_Z STR_a STR_n STR_a STR_b STR_a STR_z STR_a STR_r STR_UNDERSCORE STR_S STR_q STR_u STR_a STR_r STR_e "\0" @@ -460,6 +468,7 @@ const char PRIV(utt_names)[] = STRING_Chakma0 STRING_Cham0 STRING_Cherokee0 + STRING_Chorasmian0 STRING_Cn0 STRING_Co0 STRING_Common0 @@ -470,10 +479,12 @@ const char PRIV(utt_names)[] = STRING_Cyrillic0 STRING_Deseret0 STRING_Devanagari0 + STRING_Dives_Akuru0 STRING_Dogra0 STRING_Duployan0 STRING_Egyptian_Hieroglyphs0 STRING_Elbasan0 + STRING_Elymaic0 STRING_Ethiopic0 STRING_Georgian0 STRING_Glagolitic0 @@ -500,6 +511,7 @@ const char PRIV(utt_names)[] = STRING_Katakana0 STRING_Kayah_Li0 STRING_Kharoshthi0 + STRING_Khitan_Small_Script0 STRING_Khmer0 STRING_Khojki0 STRING_Khudawadi0 @@ -543,6 +555,7 @@ const char PRIV(utt_names)[] = STRING_Myanmar0 STRING_N0 STRING_Nabataean0 + STRING_Nandinagari0 STRING_Nd0 STRING_New_Tai_Lue0 STRING_Newa0 @@ -550,6 +563,7 @@ const char PRIV(utt_names)[] = STRING_Nl0 STRING_No0 STRING_Nushu0 + STRING_Nyiakeng_Puachue_Hmong0 STRING_Ogham0 STRING_Ol_Chiki0 STRING_Old_Hungarian0 @@ -614,12 +628,14 @@ const char PRIV(utt_names)[] = STRING_Ugaritic0 STRING_Unknown0 STRING_Vai0 + STRING_Wancho0 STRING_Warang_Citi0 STRING_Xan0 STRING_Xps0 STRING_Xsp0 STRING_Xuc0 STRING_Xwd0 + STRING_Yezidi0 STRING_Yi0 STRING_Z0 STRING_Zanabazar_Square0 @@ -655,172 +671,180 @@ const ucp_type_table PRIV(utt)[] = { { 203, PT_SC, ucp_Chakma }, { 210, PT_SC, ucp_Cham }, { 215, PT_SC, ucp_Cherokee }, - { 224, PT_PC, ucp_Cn }, - { 227, PT_PC, ucp_Co }, - { 230, PT_SC, ucp_Common }, - { 237, PT_SC, ucp_Coptic }, - { 244, PT_PC, ucp_Cs }, - { 247, PT_SC, ucp_Cuneiform }, - { 257, PT_SC, ucp_Cypriot }, - { 265, PT_SC, ucp_Cyrillic }, - { 274, PT_SC, ucp_Deseret }, - { 282, PT_SC, ucp_Devanagari }, - { 293, PT_SC, ucp_Dogra }, - { 299, PT_SC, ucp_Duployan }, - { 308, PT_SC, ucp_Egyptian_Hieroglyphs }, - { 329, PT_SC, ucp_Elbasan }, - { 337, PT_SC, ucp_Ethiopic }, - { 346, PT_SC, ucp_Georgian }, - { 355, PT_SC, ucp_Glagolitic }, - { 366, PT_SC, ucp_Gothic }, - { 373, PT_SC, ucp_Grantha }, - { 381, PT_SC, ucp_Greek }, - { 387, PT_SC, ucp_Gujarati }, - { 396, PT_SC, ucp_Gunjala_Gondi }, - { 410, PT_SC, ucp_Gurmukhi }, - { 419, PT_SC, ucp_Han }, - { 423, PT_SC, ucp_Hangul }, - { 430, PT_SC, ucp_Hanifi_Rohingya }, - { 446, PT_SC, ucp_Hanunoo }, - { 454, PT_SC, ucp_Hatran }, - { 461, PT_SC, ucp_Hebrew }, - { 468, PT_SC, ucp_Hiragana }, - { 477, PT_SC, ucp_Imperial_Aramaic }, - { 494, PT_SC, ucp_Inherited }, - { 504, PT_SC, ucp_Inscriptional_Pahlavi }, - { 526, PT_SC, ucp_Inscriptional_Parthian }, - { 549, PT_SC, ucp_Javanese }, - { 558, PT_SC, ucp_Kaithi }, - { 565, PT_SC, ucp_Kannada }, - { 573, PT_SC, ucp_Katakana }, - { 582, PT_SC, ucp_Kayah_Li }, - { 591, PT_SC, ucp_Kharoshthi }, - { 602, PT_SC, ucp_Khmer }, - { 608, PT_SC, ucp_Khojki }, - { 615, PT_SC, ucp_Khudawadi }, - { 625, PT_GC, ucp_L }, - { 627, PT_LAMP, 0 }, - { 630, PT_SC, ucp_Lao }, - { 634, PT_SC, ucp_Latin }, - { 640, PT_SC, ucp_Lepcha }, - { 647, PT_SC, ucp_Limbu }, - { 653, PT_SC, ucp_Linear_A }, - { 662, PT_SC, ucp_Linear_B }, - { 671, PT_SC, ucp_Lisu }, - { 676, PT_PC, ucp_Ll }, - { 679, PT_PC, ucp_Lm }, - { 682, PT_PC, ucp_Lo }, - { 685, PT_PC, ucp_Lt }, - { 688, PT_PC, ucp_Lu }, - { 691, PT_SC, ucp_Lycian }, - { 698, PT_SC, ucp_Lydian }, - { 705, PT_GC, ucp_M }, - { 707, PT_SC, ucp_Mahajani }, - { 716, PT_SC, ucp_Makasar }, - { 724, PT_SC, ucp_Malayalam }, - { 734, PT_SC, ucp_Mandaic }, - { 742, PT_SC, ucp_Manichaean }, - { 753, PT_SC, ucp_Marchen }, - { 761, PT_SC, ucp_Masaram_Gondi }, - { 775, PT_PC, ucp_Mc }, - { 778, PT_PC, ucp_Me }, - { 781, PT_SC, ucp_Medefaidrin }, - { 793, PT_SC, ucp_Meetei_Mayek }, - { 806, PT_SC, ucp_Mende_Kikakui }, - { 820, PT_SC, ucp_Meroitic_Cursive }, - { 837, PT_SC, ucp_Meroitic_Hieroglyphs }, - { 858, PT_SC, ucp_Miao }, - { 863, PT_PC, ucp_Mn }, - { 866, PT_SC, ucp_Modi }, - { 871, PT_SC, ucp_Mongolian }, - { 881, PT_SC, ucp_Mro }, - { 885, PT_SC, ucp_Multani }, - { 893, PT_SC, ucp_Myanmar }, - { 901, PT_GC, ucp_N }, - { 903, PT_SC, ucp_Nabataean }, - { 913, PT_PC, ucp_Nd }, - { 916, PT_SC, ucp_New_Tai_Lue }, - { 928, PT_SC, ucp_Newa }, - { 933, PT_SC, ucp_Nko }, - { 937, PT_PC, ucp_Nl }, - { 940, PT_PC, ucp_No }, - { 943, PT_SC, ucp_Nushu }, - { 949, PT_SC, ucp_Ogham }, - { 955, PT_SC, ucp_Ol_Chiki }, - { 964, PT_SC, ucp_Old_Hungarian }, - { 978, PT_SC, ucp_Old_Italic }, - { 989, PT_SC, ucp_Old_North_Arabian }, - { 1007, PT_SC, ucp_Old_Permic }, - { 1018, PT_SC, ucp_Old_Persian }, - { 1030, PT_SC, ucp_Old_Sogdian }, - { 1042, PT_SC, ucp_Old_South_Arabian }, - { 1060, PT_SC, ucp_Old_Turkic }, - { 1071, PT_SC, ucp_Oriya }, - { 1077, PT_SC, ucp_Osage }, - { 1083, PT_SC, ucp_Osmanya }, - { 1091, PT_GC, ucp_P }, - { 1093, PT_SC, ucp_Pahawh_Hmong }, - { 1106, PT_SC, ucp_Palmyrene }, - { 1116, PT_SC, ucp_Pau_Cin_Hau }, - { 1128, PT_PC, ucp_Pc }, - { 1131, PT_PC, ucp_Pd }, - { 1134, PT_PC, ucp_Pe }, - { 1137, PT_PC, ucp_Pf }, - { 1140, PT_SC, ucp_Phags_Pa }, - { 1149, PT_SC, ucp_Phoenician }, - { 1160, PT_PC, ucp_Pi }, - { 1163, PT_PC, ucp_Po }, - { 1166, PT_PC, ucp_Ps }, - { 1169, PT_SC, ucp_Psalter_Pahlavi }, - { 1185, PT_SC, ucp_Rejang }, - { 1192, PT_SC, ucp_Runic }, - { 1198, PT_GC, ucp_S }, - { 1200, PT_SC, ucp_Samaritan }, - { 1210, PT_SC, ucp_Saurashtra }, - { 1221, PT_PC, ucp_Sc }, - { 1224, PT_SC, ucp_Sharada }, - { 1232, PT_SC, ucp_Shavian }, - { 1240, PT_SC, ucp_Siddham }, - { 1248, PT_SC, ucp_SignWriting }, - { 1260, PT_SC, ucp_Sinhala }, - { 1268, PT_PC, ucp_Sk }, - { 1271, PT_PC, ucp_Sm }, - { 1274, PT_PC, ucp_So }, - { 1277, PT_SC, ucp_Sogdian }, - { 1285, PT_SC, ucp_Sora_Sompeng }, - { 1298, PT_SC, ucp_Soyombo }, - { 1306, PT_SC, ucp_Sundanese }, - { 1316, PT_SC, ucp_Syloti_Nagri }, - { 1329, PT_SC, ucp_Syriac }, - { 1336, PT_SC, ucp_Tagalog }, - { 1344, PT_SC, ucp_Tagbanwa }, - { 1353, PT_SC, ucp_Tai_Le }, - { 1360, PT_SC, ucp_Tai_Tham }, - { 1369, PT_SC, ucp_Tai_Viet }, - { 1378, PT_SC, ucp_Takri }, - { 1384, PT_SC, ucp_Tamil }, - { 1390, PT_SC, ucp_Tangut }, - { 1397, PT_SC, ucp_Telugu }, - { 1404, PT_SC, ucp_Thaana }, - { 1411, PT_SC, ucp_Thai }, - { 1416, PT_SC, ucp_Tibetan }, - { 1424, PT_SC, ucp_Tifinagh }, - { 1433, PT_SC, ucp_Tirhuta }, - { 1441, PT_SC, ucp_Ugaritic }, - { 1450, PT_SC, ucp_Unknown }, - { 1458, PT_SC, ucp_Vai }, - { 1462, PT_SC, ucp_Warang_Citi }, - { 1474, PT_ALNUM, 0 }, - { 1478, PT_PXSPACE, 0 }, - { 1482, PT_SPACE, 0 }, - { 1486, PT_UCNC, 0 }, - { 1490, PT_WORD, 0 }, - { 1494, PT_SC, ucp_Yi }, - { 1497, PT_GC, ucp_Z }, - { 1499, PT_SC, ucp_Zanabazar_Square }, - { 1516, PT_PC, ucp_Zl }, - { 1519, PT_PC, ucp_Zp }, - { 1522, PT_PC, ucp_Zs } + { 224, PT_SC, ucp_Chorasmian }, + { 235, PT_PC, ucp_Cn }, + { 238, PT_PC, ucp_Co }, + { 241, PT_SC, ucp_Common }, + { 248, PT_SC, ucp_Coptic }, + { 255, PT_PC, ucp_Cs }, + { 258, PT_SC, ucp_Cuneiform }, + { 268, PT_SC, ucp_Cypriot }, + { 276, PT_SC, ucp_Cyrillic }, + { 285, PT_SC, ucp_Deseret }, + { 293, PT_SC, ucp_Devanagari }, + { 304, PT_SC, ucp_Dives_Akuru }, + { 316, PT_SC, ucp_Dogra }, + { 322, PT_SC, ucp_Duployan }, + { 331, PT_SC, ucp_Egyptian_Hieroglyphs }, + { 352, PT_SC, ucp_Elbasan }, + { 360, PT_SC, ucp_Elymaic }, + { 368, PT_SC, ucp_Ethiopic }, + { 377, PT_SC, ucp_Georgian }, + { 386, PT_SC, ucp_Glagolitic }, + { 397, PT_SC, ucp_Gothic }, + { 404, PT_SC, ucp_Grantha }, + { 412, PT_SC, ucp_Greek }, + { 418, PT_SC, ucp_Gujarati }, + { 427, PT_SC, ucp_Gunjala_Gondi }, + { 441, PT_SC, ucp_Gurmukhi }, + { 450, PT_SC, ucp_Han }, + { 454, PT_SC, ucp_Hangul }, + { 461, PT_SC, ucp_Hanifi_Rohingya }, + { 477, PT_SC, ucp_Hanunoo }, + { 485, PT_SC, ucp_Hatran }, + { 492, PT_SC, ucp_Hebrew }, + { 499, PT_SC, ucp_Hiragana }, + { 508, PT_SC, ucp_Imperial_Aramaic }, + { 525, PT_SC, ucp_Inherited }, + { 535, PT_SC, ucp_Inscriptional_Pahlavi }, + { 557, PT_SC, ucp_Inscriptional_Parthian }, + { 580, PT_SC, ucp_Javanese }, + { 589, PT_SC, ucp_Kaithi }, + { 596, PT_SC, ucp_Kannada }, + { 604, PT_SC, ucp_Katakana }, + { 613, PT_SC, ucp_Kayah_Li }, + { 622, PT_SC, ucp_Kharoshthi }, + { 633, PT_SC, ucp_Khitan_Small_Script }, + { 653, PT_SC, ucp_Khmer }, + { 659, PT_SC, ucp_Khojki }, + { 666, PT_SC, ucp_Khudawadi }, + { 676, PT_GC, ucp_L }, + { 678, PT_LAMP, 0 }, + { 681, PT_SC, ucp_Lao }, + { 685, PT_SC, ucp_Latin }, + { 691, PT_SC, ucp_Lepcha }, + { 698, PT_SC, ucp_Limbu }, + { 704, PT_SC, ucp_Linear_A }, + { 713, PT_SC, ucp_Linear_B }, + { 722, PT_SC, ucp_Lisu }, + { 727, PT_PC, ucp_Ll }, + { 730, PT_PC, ucp_Lm }, + { 733, PT_PC, ucp_Lo }, + { 736, PT_PC, ucp_Lt }, + { 739, PT_PC, ucp_Lu }, + { 742, PT_SC, ucp_Lycian }, + { 749, PT_SC, ucp_Lydian }, + { 756, PT_GC, ucp_M }, + { 758, PT_SC, ucp_Mahajani }, + { 767, PT_SC, ucp_Makasar }, + { 775, PT_SC, ucp_Malayalam }, + { 785, PT_SC, ucp_Mandaic }, + { 793, PT_SC, ucp_Manichaean }, + { 804, PT_SC, ucp_Marchen }, + { 812, PT_SC, ucp_Masaram_Gondi }, + { 826, PT_PC, ucp_Mc }, + { 829, PT_PC, ucp_Me }, + { 832, PT_SC, ucp_Medefaidrin }, + { 844, PT_SC, ucp_Meetei_Mayek }, + { 857, PT_SC, ucp_Mende_Kikakui }, + { 871, PT_SC, ucp_Meroitic_Cursive }, + { 888, PT_SC, ucp_Meroitic_Hieroglyphs }, + { 909, PT_SC, ucp_Miao }, + { 914, PT_PC, ucp_Mn }, + { 917, PT_SC, ucp_Modi }, + { 922, PT_SC, ucp_Mongolian }, + { 932, PT_SC, ucp_Mro }, + { 936, PT_SC, ucp_Multani }, + { 944, PT_SC, ucp_Myanmar }, + { 952, PT_GC, ucp_N }, + { 954, PT_SC, ucp_Nabataean }, + { 964, PT_SC, ucp_Nandinagari }, + { 976, PT_PC, ucp_Nd }, + { 979, PT_SC, ucp_New_Tai_Lue }, + { 991, PT_SC, ucp_Newa }, + { 996, PT_SC, ucp_Nko }, + { 1000, PT_PC, ucp_Nl }, + { 1003, PT_PC, ucp_No }, + { 1006, PT_SC, ucp_Nushu }, + { 1012, PT_SC, ucp_Nyiakeng_Puachue_Hmong }, + { 1035, PT_SC, ucp_Ogham }, + { 1041, PT_SC, ucp_Ol_Chiki }, + { 1050, PT_SC, ucp_Old_Hungarian }, + { 1064, PT_SC, ucp_Old_Italic }, + { 1075, PT_SC, ucp_Old_North_Arabian }, + { 1093, PT_SC, ucp_Old_Permic }, + { 1104, PT_SC, ucp_Old_Persian }, + { 1116, PT_SC, ucp_Old_Sogdian }, + { 1128, PT_SC, ucp_Old_South_Arabian }, + { 1146, PT_SC, ucp_Old_Turkic }, + { 1157, PT_SC, ucp_Oriya }, + { 1163, PT_SC, ucp_Osage }, + { 1169, PT_SC, ucp_Osmanya }, + { 1177, PT_GC, ucp_P }, + { 1179, PT_SC, ucp_Pahawh_Hmong }, + { 1192, PT_SC, ucp_Palmyrene }, + { 1202, PT_SC, ucp_Pau_Cin_Hau }, + { 1214, PT_PC, ucp_Pc }, + { 1217, PT_PC, ucp_Pd }, + { 1220, PT_PC, ucp_Pe }, + { 1223, PT_PC, ucp_Pf }, + { 1226, PT_SC, ucp_Phags_Pa }, + { 1235, PT_SC, ucp_Phoenician }, + { 1246, PT_PC, ucp_Pi }, + { 1249, PT_PC, ucp_Po }, + { 1252, PT_PC, ucp_Ps }, + { 1255, PT_SC, ucp_Psalter_Pahlavi }, + { 1271, PT_SC, ucp_Rejang }, + { 1278, PT_SC, ucp_Runic }, + { 1284, PT_GC, ucp_S }, + { 1286, PT_SC, ucp_Samaritan }, + { 1296, PT_SC, ucp_Saurashtra }, + { 1307, PT_PC, ucp_Sc }, + { 1310, PT_SC, ucp_Sharada }, + { 1318, PT_SC, ucp_Shavian }, + { 1326, PT_SC, ucp_Siddham }, + { 1334, PT_SC, ucp_SignWriting }, + { 1346, PT_SC, ucp_Sinhala }, + { 1354, PT_PC, ucp_Sk }, + { 1357, PT_PC, ucp_Sm }, + { 1360, PT_PC, ucp_So }, + { 1363, PT_SC, ucp_Sogdian }, + { 1371, PT_SC, ucp_Sora_Sompeng }, + { 1384, PT_SC, ucp_Soyombo }, + { 1392, PT_SC, ucp_Sundanese }, + { 1402, PT_SC, ucp_Syloti_Nagri }, + { 1415, PT_SC, ucp_Syriac }, + { 1422, PT_SC, ucp_Tagalog }, + { 1430, PT_SC, ucp_Tagbanwa }, + { 1439, PT_SC, ucp_Tai_Le }, + { 1446, PT_SC, ucp_Tai_Tham }, + { 1455, PT_SC, ucp_Tai_Viet }, + { 1464, PT_SC, ucp_Takri }, + { 1470, PT_SC, ucp_Tamil }, + { 1476, PT_SC, ucp_Tangut }, + { 1483, PT_SC, ucp_Telugu }, + { 1490, PT_SC, ucp_Thaana }, + { 1497, PT_SC, ucp_Thai }, + { 1502, PT_SC, ucp_Tibetan }, + { 1510, PT_SC, ucp_Tifinagh }, + { 1519, PT_SC, ucp_Tirhuta }, + { 1527, PT_SC, ucp_Ugaritic }, + { 1536, PT_SC, ucp_Unknown }, + { 1544, PT_SC, ucp_Vai }, + { 1548, PT_SC, ucp_Wancho }, + { 1555, PT_SC, ucp_Warang_Citi }, + { 1567, PT_ALNUM, 0 }, + { 1571, PT_PXSPACE, 0 }, + { 1575, PT_SPACE, 0 }, + { 1579, PT_UCNC, 0 }, + { 1583, PT_WORD, 0 }, + { 1587, PT_SC, ucp_Yezidi }, + { 1594, PT_SC, ucp_Yi }, + { 1597, PT_GC, ucp_Z }, + { 1599, PT_SC, ucp_Zanabazar_Square }, + { 1616, PT_PC, ucp_Zl }, + { 1619, PT_PC, ucp_Zp }, + { 1622, PT_PC, ucp_Zs } }; const size_t PRIV(utt_size) = sizeof(PRIV(utt)) / sizeof(ucp_type_table); diff --git a/dist2/src/pcre2_ucd.c b/dist2/src/pcre2_ucd.c index cc53c240..46e23ff0 100644 --- a/dist2/src/pcre2_ucd.c +++ b/dist2/src/pcre2_ucd.c @@ -20,7 +20,7 @@ needed. */ /* Unicode character database. */ /* This file was autogenerated by the MultiStage2.py script. */ -/* Total size: 97152 bytes, block size: 128. */ +/* Total size: 101044 bytes, block size: 128. */ /* The tables herein are needed only when UCP support is built, and in PCRE2 that happens automatically with UTF support. @@ -39,7 +39,7 @@ const uint16_t PRIV(ucd_stage2)[] = {0}; const uint32_t PRIV(ucd_caseless_sets)[] = {0}; #else -const char *PRIV(unicode_version) = "11.0.0"; +const char *PRIV(unicode_version) = "13.0.0"; /* If the 32-bit library is run in non-32-bit mode, character values greater than 0x10ffff may be encountered. For these we set up a @@ -116,15 +116,16 @@ set of decimal digits. It is used to ensure that all the digits in a script run come from the same set. */ const uint32_t PRIV(ucd_digit_sets)[] = { - 61, /* Number of subsequent values */ + 65, /* Number of subsequent values */ 0x00039, 0x00669, 0x006f9, 0x007c9, 0x0096f, 0x009ef, 0x00a6f, 0x00aef, 0x00b6f, 0x00bef, 0x00c6f, 0x00cef, 0x00d6f, 0x00def, 0x00e59, 0x00ed9, 0x00f29, 0x01049, 0x01099, 0x017e9, 0x01819, 0x0194f, 0x019d9, 0x01a89, 0x01a99, 0x01b59, 0x01bb9, 0x01c49, 0x01c59, 0x0a629, 0x0a8d9, 0x0a909, 0x0a9d9, 0x0a9f9, 0x0aa59, 0x0abf9, 0x0ff19, 0x104a9, 0x10d39, 0x1106f, 0x110f9, 0x1113f, 0x111d9, 0x112f9, 0x11459, 0x114d9, 0x11659, 0x116c9, - 0x11739, 0x118e9, 0x11c59, 0x11d59, 0x11da9, 0x16a69, 0x16b59, 0x1d7d7, - 0x1d7e1, 0x1d7eb, 0x1d7f5, 0x1d7ff, 0x1e959, + 0x11739, 0x118e9, 0x11959, 0x11c59, 0x11d59, 0x11da9, 0x16a69, 0x16b59, + 0x1d7d7, 0x1d7e1, 0x1d7eb, 0x1d7f5, 0x1d7ff, 0x1e149, 0x1e2f9, 0x1e959, + 0x1fbf9, }; /* This vector is a list of lists of scripts for the Script Extension @@ -136,47 +137,53 @@ const uint8_t PRIV(ucd_script_sets)[] = { /* 4 */ 1, 144, 0, /* 7 */ 1, 50, 0, /* 10 */ 1, 56, 0, - /* 13 */ 2, 17, 0, - /* 16 */ 3, 15, 0, - /* 19 */ 4, 23, 0, - /* 22 */ 6, 84, 0, - /* 25 */ 12, 36, 0, - /* 28 */ 13, 18, 0, - /* 31 */ 13, 34, 0, - /* 34 */ 13, 118, 0, + /* 13 */ 3, 15, 0, + /* 16 */ 4, 23, 0, + /* 19 */ 6, 84, 0, + /* 22 */ 12, 36, 0, + /* 25 */ 13, 18, 0, + /* 28 */ 13, 34, 0, + /* 31 */ 13, 118, 0, + /* 34 */ 13, 50, 0, /* 37 */ 15, 107, 0, - /* 40 */ 15, 100, 0, - /* 43 */ 15, 54, 0, - /* 46 */ 17, 34, 0, - /* 49 */ 107, 54, 0, - /* 52 */ 21, 108, 0, - /* 55 */ 22, 129, 0, - /* 58 */ 27, 30, 0, - /* 61 */ 38, 65, 0, - /* 64 */ 1, 50, 56, 0, - /* 68 */ 3, 96, 49, 0, - /* 72 */ 96, 39, 53, 0, - /* 76 */ 12, 110, 36, 0, - /* 80 */ 15, 107, 29, 0, - /* 84 */ 15, 107, 34, 0, - /* 88 */ 23, 27, 30, 0, - /* 92 */ 69, 34, 39, 0, - /* 96 */ 1, 144, 50, 56, 0, - /* 101 */ 3, 15, 107, 29, 0, - /* 106 */ 7, 25, 52, 51, 0, - /* 111 */ 15, 142, 85, 111, 0, - /* 116 */ 4, 24, 23, 27, 30, 0, - /* 122 */ 4, 24, 23, 27, 30, 61, 0, - /* 129 */ 15, 29, 37, 44, 54, 55, 0, - /* 136 */ 132, 1, 95, 112, 121, 144, 148, 50, 0, - /* 145 */ 15, 142, 21, 22, 108, 85, 111, 114, 109, 102, 124, 0, - /* 157 */ 3, 15, 107, 21, 22, 29, 34, 37, 44, 54, 55, 124, 0, - /* 170 */ 15, 142, 21, 22, 108, 29, 85, 111, 114, 109, 102, 124, 0, - /* 183 */ 3, 15, 107, 21, 22, 29, 34, 37, 44, 100, 54, 55, 124, 0, - /* 197 */ 15, 142, 21, 22, 108, 29, 85, 111, 37, 114, 109, 102, 124, 0, - /* 211 */ 3, 15, 142, 143, 107, 21, 22, 29, 111, 37, 44, 109, 48, 49, 102, 54, 55, 124, 0, - /* 230 */ 3, 15, 142, 143, 107, 21, 22, 29, 35, 111, 37, 44, 109, 48, 49, 102, 54, 55, 124, 0, - /* 250 */ + /* 40 */ 15, 150, 0, + /* 43 */ 15, 100, 0, + /* 46 */ 15, 54, 0, + /* 49 */ 17, 34, 0, + /* 52 */ 107, 54, 0, + /* 55 */ 21, 108, 0, + /* 58 */ 22, 129, 0, + /* 61 */ 23, 34, 0, + /* 64 */ 27, 30, 0, + /* 67 */ 29, 150, 0, + /* 70 */ 34, 38, 0, + /* 73 */ 38, 65, 0, + /* 76 */ 1, 50, 56, 0, + /* 80 */ 1, 56, 156, 0, + /* 84 */ 3, 96, 49, 0, + /* 88 */ 96, 39, 53, 0, + /* 92 */ 12, 110, 36, 0, + /* 96 */ 15, 107, 29, 0, + /* 100 */ 15, 107, 34, 0, + /* 104 */ 23, 27, 30, 0, + /* 108 */ 69, 34, 39, 0, + /* 112 */ 3, 15, 107, 29, 0, + /* 117 */ 7, 25, 52, 51, 0, + /* 122 */ 15, 142, 85, 111, 0, + /* 127 */ 1, 144, 50, 56, 156, 0, + /* 133 */ 4, 24, 23, 27, 30, 0, + /* 139 */ 4, 24, 23, 27, 30, 61, 0, + /* 146 */ 15, 29, 37, 44, 54, 55, 0, + /* 153 */ 132, 1, 95, 112, 121, 144, 148, 50, 0, + /* 162 */ 3, 15, 107, 29, 150, 44, 55, 124, 0, + /* 171 */ 15, 142, 21, 22, 108, 85, 111, 114, 109, 102, 124, 0, + /* 183 */ 3, 15, 107, 21, 22, 29, 34, 37, 44, 54, 55, 124, 0, + /* 196 */ 3, 15, 107, 21, 22, 29, 34, 37, 44, 100, 54, 55, 124, 0, + /* 210 */ 15, 142, 21, 22, 108, 29, 85, 111, 114, 150, 109, 102, 124, 0, + /* 224 */ 15, 142, 21, 22, 108, 29, 85, 111, 37, 114, 150, 109, 102, 124, 0, + /* 239 */ 3, 15, 142, 143, 138, 107, 21, 22, 29, 111, 37, 150, 44, 109, 48, 49, 102, 54, 55, 124, 0, + /* 260 */ 3, 15, 142, 143, 138, 107, 21, 22, 29, 35, 111, 37, 150, 44, 109, 48, 49, 102, 54, 55, 124, 0, + /* 282 */ }; /* These are the main two-stage UCD tables. The fields in each record are: @@ -185,7 +192,7 @@ offset to multichar other cases or zero (8 bits), offset to other case or zero (32 bits, signed), script extension (16 bits, signed), and a dummy 16-bit field to make the whole thing a multiple of 4 bytes. */ -const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ +const ucd_record PRIV(ucd_records)[] = { /* 11700 bytes, record size 12 */ { 10, 0, 2, 0, 0, 10, 256, }, /* 0 */ { 10, 0, 2, 0, 0, 10, 0, }, /* 1 */ { 10, 0, 1, 0, 0, 10, 0, }, /* 2 */ @@ -288,112 +295,112 @@ const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ { 34, 5, 12, 0, -214, 34, 0, }, /* 99 */ { 34, 5, 12, 0, 10727, 34, 0, }, /* 100 */ { 34, 5, 12, 0, -218, 34, 0, }, /* 101 */ - { 34, 5, 12, 0, 42282, 34, 0, }, /* 102 */ - { 34, 5, 12, 0, -69, 34, 0, }, /* 103 */ - { 34, 5, 12, 0, -217, 34, 0, }, /* 104 */ - { 34, 5, 12, 0, -71, 34, 0, }, /* 105 */ - { 34, 5, 12, 0, -219, 34, 0, }, /* 106 */ - { 34, 5, 12, 0, 42261, 34, 0, }, /* 107 */ - { 34, 5, 12, 0, 42258, 34, 0, }, /* 108 */ - { 34, 6, 12, 0, 0, 34, 0, }, /* 109 */ - { 10, 6, 12, 0, 0, 10, 0, }, /* 110 */ - { 4, 24, 12, 0, 0, 4, 0, }, /* 111 */ - { 28, 12, 3, 0, 0, 28, 0, }, /* 112 */ - { 28, 12, 3, 0, 0, 20, 0, }, /* 113 */ - { 28, 12, 3, 21, 116, 20, 0, }, /* 114 */ - { 28, 12, 3, 0, 0, 34, 0, }, /* 115 */ - { 20, 9, 12, 0, 1, 20, 0, }, /* 116 */ - { 20, 5, 12, 0, -1, 20, 0, }, /* 117 */ - { 20, 24, 12, 0, 0, 20, 0, }, /* 118 */ - { 0, 2, 12, 0, 0, 0, 0, }, /* 119 */ - { 20, 6, 12, 0, 0, 20, 0, }, /* 120 */ - { 20, 5, 12, 0, 130, 20, 0, }, /* 121 */ - { 20, 9, 12, 0, 116, 20, 0, }, /* 122 */ - { 20, 9, 12, 0, 38, 20, 0, }, /* 123 */ - { 20, 9, 12, 0, 37, 20, 0, }, /* 124 */ - { 20, 9, 12, 0, 64, 20, 0, }, /* 125 */ - { 20, 9, 12, 0, 63, 20, 0, }, /* 126 */ - { 20, 5, 12, 0, 0, 20, 0, }, /* 127 */ - { 20, 9, 12, 0, 32, 20, 0, }, /* 128 */ - { 20, 9, 12, 34, 32, 20, 0, }, /* 129 */ - { 20, 9, 12, 59, 32, 20, 0, }, /* 130 */ - { 20, 9, 12, 38, 32, 20, 0, }, /* 131 */ - { 20, 9, 12, 21, 32, 20, 0, }, /* 132 */ - { 20, 9, 12, 51, 32, 20, 0, }, /* 133 */ - { 20, 9, 12, 26, 32, 20, 0, }, /* 134 */ - { 20, 9, 12, 47, 32, 20, 0, }, /* 135 */ - { 20, 9, 12, 55, 32, 20, 0, }, /* 136 */ - { 20, 9, 12, 30, 32, 20, 0, }, /* 137 */ - { 20, 9, 12, 43, 32, 20, 0, }, /* 138 */ - { 20, 9, 12, 96, 32, 20, 0, }, /* 139 */ - { 20, 5, 12, 0, -38, 20, 0, }, /* 140 */ - { 20, 5, 12, 0, -37, 20, 0, }, /* 141 */ - { 20, 5, 12, 0, -32, 20, 0, }, /* 142 */ - { 20, 5, 12, 34, -32, 20, 0, }, /* 143 */ - { 20, 5, 12, 59, -32, 20, 0, }, /* 144 */ - { 20, 5, 12, 38, -32, 20, 0, }, /* 145 */ - { 20, 5, 12, 21, -116, 20, 0, }, /* 146 */ - { 20, 5, 12, 51, -32, 20, 0, }, /* 147 */ - { 20, 5, 12, 26, -775, 20, 0, }, /* 148 */ - { 20, 5, 12, 47, -32, 20, 0, }, /* 149 */ - { 20, 5, 12, 55, -32, 20, 0, }, /* 150 */ - { 20, 5, 12, 30, 1, 20, 0, }, /* 151 */ - { 20, 5, 12, 30, -32, 20, 0, }, /* 152 */ - { 20, 5, 12, 43, -32, 20, 0, }, /* 153 */ - { 20, 5, 12, 96, -32, 20, 0, }, /* 154 */ - { 20, 5, 12, 0, -64, 20, 0, }, /* 155 */ - { 20, 5, 12, 0, -63, 20, 0, }, /* 156 */ - { 20, 9, 12, 0, 8, 20, 0, }, /* 157 */ - { 20, 5, 12, 34, -30, 20, 0, }, /* 158 */ - { 20, 5, 12, 38, -25, 20, 0, }, /* 159 */ - { 20, 9, 12, 0, 0, 20, 0, }, /* 160 */ - { 20, 5, 12, 43, -15, 20, 0, }, /* 161 */ - { 20, 5, 12, 47, -22, 20, 0, }, /* 162 */ - { 20, 5, 12, 0, -8, 20, 0, }, /* 163 */ - { 11, 9, 12, 0, 1, 11, 0, }, /* 164 */ - { 11, 5, 12, 0, -1, 11, 0, }, /* 165 */ - { 20, 5, 12, 51, -54, 20, 0, }, /* 166 */ - { 20, 5, 12, 55, -48, 20, 0, }, /* 167 */ - { 20, 5, 12, 0, 7, 20, 0, }, /* 168 */ - { 20, 5, 12, 0, -116, 20, 0, }, /* 169 */ - { 20, 9, 12, 38, -60, 20, 0, }, /* 170 */ - { 20, 5, 12, 59, -64, 20, 0, }, /* 171 */ - { 20, 25, 12, 0, 0, 20, 0, }, /* 172 */ - { 20, 9, 12, 0, -7, 20, 0, }, /* 173 */ - { 20, 9, 12, 0, -130, 20, 0, }, /* 174 */ - { 13, 9, 12, 0, 80, 13, 0, }, /* 175 */ - { 13, 9, 12, 0, 32, 13, 0, }, /* 176 */ - { 13, 9, 12, 63, 32, 13, 0, }, /* 177 */ - { 13, 9, 12, 67, 32, 13, 0, }, /* 178 */ - { 13, 9, 12, 71, 32, 13, 0, }, /* 179 */ - { 13, 9, 12, 75, 32, 13, 0, }, /* 180 */ - { 13, 9, 12, 79, 32, 13, 0, }, /* 181 */ - { 13, 9, 12, 84, 32, 13, 0, }, /* 182 */ - { 13, 5, 12, 0, -32, 13, 0, }, /* 183 */ - { 13, 5, 12, 63, -32, 13, 0, }, /* 184 */ - { 13, 5, 12, 67, -32, 13, 0, }, /* 185 */ - { 13, 5, 12, 71, -32, 13, 0, }, /* 186 */ - { 13, 5, 12, 75, -32, 13, 0, }, /* 187 */ - { 13, 5, 12, 79, -32, 13, 0, }, /* 188 */ - { 13, 5, 12, 84, -32, 13, 0, }, /* 189 */ - { 13, 5, 12, 0, -80, 13, 0, }, /* 190 */ - { 13, 9, 12, 0, 1, 13, 0, }, /* 191 */ - { 13, 5, 12, 0, -1, 13, 0, }, /* 192 */ - { 13, 9, 12, 88, 1, 13, 0, }, /* 193 */ - { 13, 5, 12, 88, -1, 13, 0, }, /* 194 */ - { 13, 26, 12, 0, 0, 13, 0, }, /* 195 */ - { 13, 12, 3, 0, 0, -34, 0, }, /* 196 */ - { 13, 12, 3, 0, 0, -28, 0, }, /* 197 */ - { 28, 12, 3, 0, 0, -31, 0, }, /* 198 */ - { 13, 11, 3, 0, 0, 13, 0, }, /* 199 */ - { 13, 9, 12, 0, 15, 13, 0, }, /* 200 */ - { 13, 5, 12, 0, -15, 13, 0, }, /* 201 */ - { 2, 9, 12, 0, 48, 2, 0, }, /* 202 */ - { 2, 6, 12, 0, 0, 2, 0, }, /* 203 */ - { 2, 21, 12, 0, 0, 2, 0, }, /* 204 */ - { 2, 5, 12, 0, 0, 2, 0, }, /* 205 */ - { 2, 5, 12, 0, -48, 2, 0, }, /* 206 */ - { 10, 21, 12, 0, 0, -13, 0, }, /* 207 */ + { 34, 5, 12, 0, 42307, 34, 0, }, /* 102 */ + { 34, 5, 12, 0, 42282, 34, 0, }, /* 103 */ + { 34, 5, 12, 0, -69, 34, 0, }, /* 104 */ + { 34, 5, 12, 0, -217, 34, 0, }, /* 105 */ + { 34, 5, 12, 0, -71, 34, 0, }, /* 106 */ + { 34, 5, 12, 0, -219, 34, 0, }, /* 107 */ + { 34, 5, 12, 0, 42261, 34, 0, }, /* 108 */ + { 34, 5, 12, 0, 42258, 34, 0, }, /* 109 */ + { 34, 6, 12, 0, 0, 34, 0, }, /* 110 */ + { 10, 6, 12, 0, 0, 10, 0, }, /* 111 */ + { 4, 24, 12, 0, 0, 4, 0, }, /* 112 */ + { 28, 12, 3, 0, 0, 28, 0, }, /* 113 */ + { 28, 12, 3, 0, 0, 20, 0, }, /* 114 */ + { 28, 12, 3, 21, 116, 20, 0, }, /* 115 */ + { 28, 12, 3, 0, 0, 34, 0, }, /* 116 */ + { 20, 9, 12, 0, 1, 20, 0, }, /* 117 */ + { 20, 5, 12, 0, -1, 20, 0, }, /* 118 */ + { 20, 24, 12, 0, 0, 20, 0, }, /* 119 */ + { 0, 2, 12, 0, 0, 0, 0, }, /* 120 */ + { 20, 6, 12, 0, 0, 20, 0, }, /* 121 */ + { 20, 5, 12, 0, 130, 20, 0, }, /* 122 */ + { 20, 9, 12, 0, 116, 20, 0, }, /* 123 */ + { 20, 9, 12, 0, 38, 20, 0, }, /* 124 */ + { 20, 9, 12, 0, 37, 20, 0, }, /* 125 */ + { 20, 9, 12, 0, 64, 20, 0, }, /* 126 */ + { 20, 9, 12, 0, 63, 20, 0, }, /* 127 */ + { 20, 5, 12, 0, 0, 20, 0, }, /* 128 */ + { 20, 9, 12, 0, 32, 20, 0, }, /* 129 */ + { 20, 9, 12, 34, 32, 20, 0, }, /* 130 */ + { 20, 9, 12, 59, 32, 20, 0, }, /* 131 */ + { 20, 9, 12, 38, 32, 20, 0, }, /* 132 */ + { 20, 9, 12, 21, 32, 20, 0, }, /* 133 */ + { 20, 9, 12, 51, 32, 20, 0, }, /* 134 */ + { 20, 9, 12, 26, 32, 20, 0, }, /* 135 */ + { 20, 9, 12, 47, 32, 20, 0, }, /* 136 */ + { 20, 9, 12, 55, 32, 20, 0, }, /* 137 */ + { 20, 9, 12, 30, 32, 20, 0, }, /* 138 */ + { 20, 9, 12, 43, 32, 20, 0, }, /* 139 */ + { 20, 9, 12, 96, 32, 20, 0, }, /* 140 */ + { 20, 5, 12, 0, -38, 20, 0, }, /* 141 */ + { 20, 5, 12, 0, -37, 20, 0, }, /* 142 */ + { 20, 5, 12, 0, -32, 20, 0, }, /* 143 */ + { 20, 5, 12, 34, -32, 20, 0, }, /* 144 */ + { 20, 5, 12, 59, -32, 20, 0, }, /* 145 */ + { 20, 5, 12, 38, -32, 20, 0, }, /* 146 */ + { 20, 5, 12, 21, -116, 20, 0, }, /* 147 */ + { 20, 5, 12, 51, -32, 20, 0, }, /* 148 */ + { 20, 5, 12, 26, -775, 20, 0, }, /* 149 */ + { 20, 5, 12, 47, -32, 20, 0, }, /* 150 */ + { 20, 5, 12, 55, -32, 20, 0, }, /* 151 */ + { 20, 5, 12, 30, 1, 20, 0, }, /* 152 */ + { 20, 5, 12, 30, -32, 20, 0, }, /* 153 */ + { 20, 5, 12, 43, -32, 20, 0, }, /* 154 */ + { 20, 5, 12, 96, -32, 20, 0, }, /* 155 */ + { 20, 5, 12, 0, -64, 20, 0, }, /* 156 */ + { 20, 5, 12, 0, -63, 20, 0, }, /* 157 */ + { 20, 9, 12, 0, 8, 20, 0, }, /* 158 */ + { 20, 5, 12, 34, -30, 20, 0, }, /* 159 */ + { 20, 5, 12, 38, -25, 20, 0, }, /* 160 */ + { 20, 9, 12, 0, 0, 20, 0, }, /* 161 */ + { 20, 5, 12, 43, -15, 20, 0, }, /* 162 */ + { 20, 5, 12, 47, -22, 20, 0, }, /* 163 */ + { 20, 5, 12, 0, -8, 20, 0, }, /* 164 */ + { 11, 9, 12, 0, 1, 11, 0, }, /* 165 */ + { 11, 5, 12, 0, -1, 11, 0, }, /* 166 */ + { 20, 5, 12, 51, -54, 20, 0, }, /* 167 */ + { 20, 5, 12, 55, -48, 20, 0, }, /* 168 */ + { 20, 5, 12, 0, 7, 20, 0, }, /* 169 */ + { 20, 5, 12, 0, -116, 20, 0, }, /* 170 */ + { 20, 9, 12, 38, -60, 20, 0, }, /* 171 */ + { 20, 5, 12, 59, -64, 20, 0, }, /* 172 */ + { 20, 25, 12, 0, 0, 20, 0, }, /* 173 */ + { 20, 9, 12, 0, -7, 20, 0, }, /* 174 */ + { 20, 9, 12, 0, -130, 20, 0, }, /* 175 */ + { 13, 9, 12, 0, 80, 13, 0, }, /* 176 */ + { 13, 9, 12, 0, 32, 13, 0, }, /* 177 */ + { 13, 9, 12, 63, 32, 13, 0, }, /* 178 */ + { 13, 9, 12, 67, 32, 13, 0, }, /* 179 */ + { 13, 9, 12, 71, 32, 13, 0, }, /* 180 */ + { 13, 9, 12, 75, 32, 13, 0, }, /* 181 */ + { 13, 9, 12, 79, 32, 13, 0, }, /* 182 */ + { 13, 9, 12, 84, 32, 13, 0, }, /* 183 */ + { 13, 5, 12, 0, -32, 13, 0, }, /* 184 */ + { 13, 5, 12, 63, -32, 13, 0, }, /* 185 */ + { 13, 5, 12, 67, -32, 13, 0, }, /* 186 */ + { 13, 5, 12, 71, -32, 13, 0, }, /* 187 */ + { 13, 5, 12, 75, -32, 13, 0, }, /* 188 */ + { 13, 5, 12, 79, -32, 13, 0, }, /* 189 */ + { 13, 5, 12, 84, -32, 13, 0, }, /* 190 */ + { 13, 5, 12, 0, -80, 13, 0, }, /* 191 */ + { 13, 9, 12, 0, 1, 13, 0, }, /* 192 */ + { 13, 5, 12, 0, -1, 13, 0, }, /* 193 */ + { 13, 9, 12, 88, 1, 13, 0, }, /* 194 */ + { 13, 5, 12, 88, -1, 13, 0, }, /* 195 */ + { 13, 26, 12, 0, 0, 13, 0, }, /* 196 */ + { 13, 12, 3, 0, 0, -31, 0, }, /* 197 */ + { 13, 12, 3, 0, 0, -25, 0, }, /* 198 */ + { 28, 12, 3, 0, 0, -28, 0, }, /* 199 */ + { 13, 11, 3, 0, 0, 13, 0, }, /* 200 */ + { 13, 9, 12, 0, 15, 13, 0, }, /* 201 */ + { 13, 5, 12, 0, -15, 13, 0, }, /* 202 */ + { 2, 9, 12, 0, 48, 2, 0, }, /* 203 */ + { 2, 6, 12, 0, 0, 2, 0, }, /* 204 */ + { 2, 21, 12, 0, 0, 2, 0, }, /* 205 */ + { 2, 5, 12, 0, 0, 2, 0, }, /* 206 */ + { 2, 5, 12, 0, -48, 2, 0, }, /* 207 */ { 2, 17, 12, 0, 0, 2, 0, }, /* 208 */ { 2, 26, 12, 0, 0, 2, 0, }, /* 209 */ { 2, 23, 12, 0, 0, 2, 0, }, /* 210 */ @@ -406,14 +413,14 @@ const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ { 1, 25, 12, 0, 0, 1, 0, }, /* 217 */ { 1, 21, 12, 0, 0, 1, 0, }, /* 218 */ { 1, 23, 12, 0, 0, 1, 0, }, /* 219 */ - { 10, 21, 12, 0, 0, -96, 0, }, /* 220 */ + { 10, 21, 12, 0, 0, -127, 0, }, /* 220 */ { 1, 26, 12, 0, 0, 1, 0, }, /* 221 */ { 1, 12, 3, 0, 0, 1, 0, }, /* 222 */ - { 1, 1, 2, 0, 0, -64, 0, }, /* 223 */ + { 1, 1, 2, 0, 0, -76, 0, }, /* 223 */ { 1, 7, 12, 0, 0, 1, 0, }, /* 224 */ - { 10, 6, 12, 0, 0, -136, 0, }, /* 225 */ + { 10, 6, 12, 0, 0, -153, 0, }, /* 225 */ { 28, 12, 3, 0, 0, -7, 0, }, /* 226 */ - { 1, 13, 12, 0, 0, -10, 0, }, /* 227 */ + { 1, 13, 12, 0, 0, -80, 0, }, /* 227 */ { 1, 21, 12, 0, 0, -4, 0, }, /* 228 */ { 1, 6, 12, 0, 0, 1, 0, }, /* 229 */ { 1, 13, 12, 0, 0, 1, 0, }, /* 230 */ @@ -440,18 +447,18 @@ const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ { 15, 12, 3, 0, 0, 15, 0, }, /* 251 */ { 15, 10, 5, 0, 0, 15, 0, }, /* 252 */ { 15, 7, 12, 0, 0, 15, 0, }, /* 253 */ - { 28, 12, 3, 0, 0, -183, 0, }, /* 254 */ - { 28, 12, 3, 0, 0, -157, 0, }, /* 255 */ - { 10, 21, 12, 0, 0, -211, 0, }, /* 256 */ - { 10, 21, 12, 0, 0, -230, 0, }, /* 257 */ - { 15, 13, 12, 0, 0, -111, 0, }, /* 258 */ + { 28, 12, 3, 0, 0, -196, 0, }, /* 254 */ + { 28, 12, 3, 0, 0, -183, 0, }, /* 255 */ + { 10, 21, 12, 0, 0, -239, 0, }, /* 256 */ + { 10, 21, 12, 0, 0, -260, 0, }, /* 257 */ + { 15, 13, 12, 0, 0, -122, 0, }, /* 258 */ { 15, 21, 12, 0, 0, 15, 0, }, /* 259 */ { 15, 6, 12, 0, 0, 15, 0, }, /* 260 */ { 3, 7, 12, 0, 0, 3, 0, }, /* 261 */ { 3, 12, 3, 0, 0, 3, 0, }, /* 262 */ { 3, 10, 5, 0, 0, 3, 0, }, /* 263 */ { 3, 10, 3, 0, 0, 3, 0, }, /* 264 */ - { 3, 13, 12, 0, 0, -68, 0, }, /* 265 */ + { 3, 13, 12, 0, 0, -84, 0, }, /* 265 */ { 3, 23, 12, 0, 0, 3, 0, }, /* 266 */ { 3, 15, 12, 0, 0, 3, 0, }, /* 267 */ { 3, 26, 12, 0, 0, 3, 0, }, /* 268 */ @@ -459,12 +466,12 @@ const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ { 22, 12, 3, 0, 0, 22, 0, }, /* 270 */ { 22, 10, 5, 0, 0, 22, 0, }, /* 271 */ { 22, 7, 12, 0, 0, 22, 0, }, /* 272 */ - { 22, 13, 12, 0, 0, -55, 0, }, /* 273 */ + { 22, 13, 12, 0, 0, -58, 0, }, /* 273 */ { 22, 21, 12, 0, 0, 22, 0, }, /* 274 */ { 21, 12, 3, 0, 0, 21, 0, }, /* 275 */ { 21, 10, 5, 0, 0, 21, 0, }, /* 276 */ { 21, 7, 12, 0, 0, 21, 0, }, /* 277 */ - { 21, 13, 12, 0, 0, -52, 0, }, /* 278 */ + { 21, 13, 12, 0, 0, -55, 0, }, /* 278 */ { 21, 21, 12, 0, 0, 21, 0, }, /* 279 */ { 21, 23, 12, 0, 0, 21, 0, }, /* 280 */ { 44, 12, 3, 0, 0, 44, 0, }, /* 281 */ @@ -478,642 +485,689 @@ const ucd_record PRIV(ucd_records)[] = { /* 11136 bytes, record size 12 */ { 54, 7, 12, 0, 0, 54, 0, }, /* 289 */ { 54, 10, 3, 0, 0, 54, 0, }, /* 290 */ { 54, 10, 5, 0, 0, 54, 0, }, /* 291 */ - { 54, 13, 12, 0, 0, -49, 0, }, /* 292 */ - { 54, 15, 12, 0, 0, -49, 0, }, /* 293 */ - { 54, 26, 12, 0, 0, -49, 0, }, /* 294 */ + { 54, 13, 12, 0, 0, -52, 0, }, /* 292 */ + { 54, 15, 12, 0, 0, -52, 0, }, /* 293 */ + { 54, 26, 12, 0, 0, -52, 0, }, /* 294 */ { 54, 26, 12, 0, 0, 54, 0, }, /* 295 */ { 54, 23, 12, 0, 0, 54, 0, }, /* 296 */ { 55, 12, 3, 0, 0, 55, 0, }, /* 297 */ { 55, 10, 5, 0, 0, 55, 0, }, /* 298 */ { 55, 7, 12, 0, 0, 55, 0, }, /* 299 */ { 55, 13, 12, 0, 0, 55, 0, }, /* 300 */ - { 55, 15, 12, 0, 0, 55, 0, }, /* 301 */ - { 55, 26, 12, 0, 0, 55, 0, }, /* 302 */ - { 29, 7, 12, 0, 0, 29, 0, }, /* 303 */ - { 29, 12, 3, 0, 0, 29, 0, }, /* 304 */ - { 29, 10, 5, 0, 0, 29, 0, }, /* 305 */ - { 29, 21, 12, 0, 0, 29, 0, }, /* 306 */ - { 29, 10, 3, 0, 0, 29, 0, }, /* 307 */ - { 29, 13, 12, 0, 0, 29, 0, }, /* 308 */ - { 37, 12, 3, 0, 0, 37, 0, }, /* 309 */ - { 37, 10, 5, 0, 0, 37, 0, }, /* 310 */ - { 37, 7, 12, 0, 0, 37, 0, }, /* 311 */ - { 37, 10, 3, 0, 0, 37, 0, }, /* 312 */ - { 37, 7, 4, 0, 0, 37, 0, }, /* 313 */ - { 37, 26, 12, 0, 0, 37, 0, }, /* 314 */ - { 37, 15, 12, 0, 0, 37, 0, }, /* 315 */ - { 37, 13, 12, 0, 0, 37, 0, }, /* 316 */ - { 48, 10, 5, 0, 0, 48, 0, }, /* 317 */ - { 48, 7, 12, 0, 0, 48, 0, }, /* 318 */ - { 48, 12, 3, 0, 0, 48, 0, }, /* 319 */ - { 48, 10, 3, 0, 0, 48, 0, }, /* 320 */ - { 48, 13, 12, 0, 0, 48, 0, }, /* 321 */ - { 48, 21, 12, 0, 0, 48, 0, }, /* 322 */ - { 57, 7, 12, 0, 0, 57, 0, }, /* 323 */ - { 57, 12, 3, 0, 0, 57, 0, }, /* 324 */ - { 57, 7, 5, 0, 0, 57, 0, }, /* 325 */ - { 57, 6, 12, 0, 0, 57, 0, }, /* 326 */ - { 57, 21, 12, 0, 0, 57, 0, }, /* 327 */ - { 57, 13, 12, 0, 0, 57, 0, }, /* 328 */ - { 33, 7, 12, 0, 0, 33, 0, }, /* 329 */ - { 33, 12, 3, 0, 0, 33, 0, }, /* 330 */ - { 33, 7, 5, 0, 0, 33, 0, }, /* 331 */ - { 33, 6, 12, 0, 0, 33, 0, }, /* 332 */ - { 33, 13, 12, 0, 0, 33, 0, }, /* 333 */ - { 58, 7, 12, 0, 0, 58, 0, }, /* 334 */ - { 58, 26, 12, 0, 0, 58, 0, }, /* 335 */ - { 58, 21, 12, 0, 0, 58, 0, }, /* 336 */ - { 58, 12, 3, 0, 0, 58, 0, }, /* 337 */ - { 58, 13, 12, 0, 0, 58, 0, }, /* 338 */ - { 58, 15, 12, 0, 0, 58, 0, }, /* 339 */ - { 58, 22, 12, 0, 0, 58, 0, }, /* 340 */ - { 58, 18, 12, 0, 0, 58, 0, }, /* 341 */ - { 58, 10, 5, 0, 0, 58, 0, }, /* 342 */ - { 39, 7, 12, 0, 0, 39, 0, }, /* 343 */ - { 39, 10, 12, 0, 0, 39, 0, }, /* 344 */ - { 39, 12, 3, 0, 0, 39, 0, }, /* 345 */ - { 39, 10, 5, 0, 0, 39, 0, }, /* 346 */ - { 39, 13, 12, 0, 0, -72, 0, }, /* 347 */ - { 39, 21, 12, 0, 0, 39, 0, }, /* 348 */ - { 39, 13, 12, 0, 0, 39, 0, }, /* 349 */ - { 39, 26, 12, 0, 0, 39, 0, }, /* 350 */ - { 17, 9, 12, 0, 7264, 17, 0, }, /* 351 */ - { 17, 5, 12, 0, 3008, 17, 0, }, /* 352 */ - { 10, 21, 12, 0, 0, -46, 0, }, /* 353 */ - { 17, 6, 12, 0, 0, 17, 0, }, /* 354 */ - { 24, 7, 6, 0, 0, 24, 0, }, /* 355 */ - { 24, 7, 7, 0, 0, 24, 0, }, /* 356 */ - { 24, 7, 8, 0, 0, 24, 0, }, /* 357 */ - { 16, 7, 12, 0, 0, 16, 0, }, /* 358 */ - { 16, 12, 3, 0, 0, 16, 0, }, /* 359 */ - { 16, 21, 12, 0, 0, 16, 0, }, /* 360 */ - { 16, 15, 12, 0, 0, 16, 0, }, /* 361 */ - { 16, 26, 12, 0, 0, 16, 0, }, /* 362 */ - { 9, 9, 12, 0, 38864, 9, 0, }, /* 363 */ - { 9, 9, 12, 0, 8, 9, 0, }, /* 364 */ - { 9, 5, 12, 0, -8, 9, 0, }, /* 365 */ - { 8, 17, 12, 0, 0, 8, 0, }, /* 366 */ - { 8, 7, 12, 0, 0, 8, 0, }, /* 367 */ - { 8, 21, 12, 0, 0, 8, 0, }, /* 368 */ - { 41, 29, 12, 0, 0, 41, 0, }, /* 369 */ - { 41, 7, 12, 0, 0, 41, 0, }, /* 370 */ - { 41, 22, 12, 0, 0, 41, 0, }, /* 371 */ - { 41, 18, 12, 0, 0, 41, 0, }, /* 372 */ - { 46, 7, 12, 0, 0, 46, 0, }, /* 373 */ - { 46, 14, 12, 0, 0, 46, 0, }, /* 374 */ - { 51, 7, 12, 0, 0, 51, 0, }, /* 375 */ - { 51, 12, 3, 0, 0, 51, 0, }, /* 376 */ - { 25, 7, 12, 0, 0, 25, 0, }, /* 377 */ - { 25, 12, 3, 0, 0, 25, 0, }, /* 378 */ - { 10, 21, 12, 0, 0, -106, 0, }, /* 379 */ - { 7, 7, 12, 0, 0, 7, 0, }, /* 380 */ - { 7, 12, 3, 0, 0, 7, 0, }, /* 381 */ - { 52, 7, 12, 0, 0, 52, 0, }, /* 382 */ - { 52, 12, 3, 0, 0, 52, 0, }, /* 383 */ - { 32, 7, 12, 0, 0, 32, 0, }, /* 384 */ - { 32, 12, 3, 0, 0, 32, 0, }, /* 385 */ - { 32, 10, 5, 0, 0, 32, 0, }, /* 386 */ - { 32, 21, 12, 0, 0, 32, 0, }, /* 387 */ - { 32, 6, 12, 0, 0, 32, 0, }, /* 388 */ - { 32, 23, 12, 0, 0, 32, 0, }, /* 389 */ - { 32, 13, 12, 0, 0, 32, 0, }, /* 390 */ - { 32, 15, 12, 0, 0, 32, 0, }, /* 391 */ - { 38, 21, 12, 0, 0, 38, 0, }, /* 392 */ - { 10, 21, 12, 0, 0, -61, 0, }, /* 393 */ - { 38, 17, 12, 0, 0, 38, 0, }, /* 394 */ - { 38, 12, 3, 0, 0, 38, 0, }, /* 395 */ - { 38, 1, 2, 0, 0, 38, 0, }, /* 396 */ - { 38, 13, 12, 0, 0, 38, 0, }, /* 397 */ - { 38, 7, 12, 0, 0, 38, 0, }, /* 398 */ - { 38, 6, 12, 0, 0, 38, 0, }, /* 399 */ - { 35, 7, 12, 0, 0, 35, 0, }, /* 400 */ - { 35, 12, 3, 0, 0, 35, 0, }, /* 401 */ - { 35, 10, 5, 0, 0, 35, 0, }, /* 402 */ - { 35, 26, 12, 0, 0, 35, 0, }, /* 403 */ - { 35, 21, 12, 0, 0, 35, 0, }, /* 404 */ - { 35, 13, 12, 0, 0, 35, 0, }, /* 405 */ - { 53, 7, 12, 0, 0, 53, 0, }, /* 406 */ - { 40, 7, 12, 0, 0, 40, 0, }, /* 407 */ - { 40, 13, 12, 0, 0, 40, 0, }, /* 408 */ - { 40, 15, 12, 0, 0, 40, 0, }, /* 409 */ - { 40, 26, 12, 0, 0, 40, 0, }, /* 410 */ - { 32, 26, 12, 0, 0, 32, 0, }, /* 411 */ - { 6, 7, 12, 0, 0, 6, 0, }, /* 412 */ - { 6, 12, 3, 0, 0, 6, 0, }, /* 413 */ - { 6, 10, 5, 0, 0, 6, 0, }, /* 414 */ - { 6, 21, 12, 0, 0, 6, 0, }, /* 415 */ - { 91, 7, 12, 0, 0, 91, 0, }, /* 416 */ - { 91, 10, 5, 0, 0, 91, 0, }, /* 417 */ - { 91, 12, 3, 0, 0, 91, 0, }, /* 418 */ - { 91, 10, 12, 0, 0, 91, 0, }, /* 419 */ - { 91, 13, 12, 0, 0, 91, 0, }, /* 420 */ - { 91, 21, 12, 0, 0, 91, 0, }, /* 421 */ - { 91, 6, 12, 0, 0, 91, 0, }, /* 422 */ - { 28, 11, 3, 0, 0, 28, 0, }, /* 423 */ - { 62, 12, 3, 0, 0, 62, 0, }, /* 424 */ - { 62, 10, 5, 0, 0, 62, 0, }, /* 425 */ - { 62, 7, 12, 0, 0, 62, 0, }, /* 426 */ - { 62, 13, 12, 0, 0, 62, 0, }, /* 427 */ - { 62, 21, 12, 0, 0, 62, 0, }, /* 428 */ - { 62, 26, 12, 0, 0, 62, 0, }, /* 429 */ - { 76, 12, 3, 0, 0, 76, 0, }, /* 430 */ - { 76, 10, 5, 0, 0, 76, 0, }, /* 431 */ - { 76, 7, 12, 0, 0, 76, 0, }, /* 432 */ - { 76, 13, 12, 0, 0, 76, 0, }, /* 433 */ - { 93, 7, 12, 0, 0, 93, 0, }, /* 434 */ - { 93, 12, 3, 0, 0, 93, 0, }, /* 435 */ - { 93, 10, 5, 0, 0, 93, 0, }, /* 436 */ - { 93, 21, 12, 0, 0, 93, 0, }, /* 437 */ - { 70, 7, 12, 0, 0, 70, 0, }, /* 438 */ - { 70, 10, 5, 0, 0, 70, 0, }, /* 439 */ - { 70, 12, 3, 0, 0, 70, 0, }, /* 440 */ - { 70, 21, 12, 0, 0, 70, 0, }, /* 441 */ - { 70, 13, 12, 0, 0, 70, 0, }, /* 442 */ - { 73, 13, 12, 0, 0, 73, 0, }, /* 443 */ - { 73, 7, 12, 0, 0, 73, 0, }, /* 444 */ - { 73, 6, 12, 0, 0, 73, 0, }, /* 445 */ - { 73, 21, 12, 0, 0, 73, 0, }, /* 446 */ - { 13, 5, 12, 63, -6222, 13, 0, }, /* 447 */ - { 13, 5, 12, 67, -6221, 13, 0, }, /* 448 */ - { 13, 5, 12, 71, -6212, 13, 0, }, /* 449 */ - { 13, 5, 12, 75, -6210, 13, 0, }, /* 450 */ - { 13, 5, 12, 79, -6210, 13, 0, }, /* 451 */ - { 13, 5, 12, 79, -6211, 13, 0, }, /* 452 */ - { 13, 5, 12, 84, -6204, 13, 0, }, /* 453 */ - { 13, 5, 12, 88, -6180, 13, 0, }, /* 454 */ - { 13, 5, 12, 108, 35267, 13, 0, }, /* 455 */ - { 17, 9, 12, 0, -3008, 17, 0, }, /* 456 */ - { 76, 21, 12, 0, 0, 76, 0, }, /* 457 */ - { 28, 12, 3, 0, 0, -101, 0, }, /* 458 */ - { 28, 12, 3, 0, 0, 15, 0, }, /* 459 */ - { 10, 21, 12, 0, 0, -37, 0, }, /* 460 */ - { 28, 12, 3, 0, 0, -16, 0, }, /* 461 */ - { 28, 12, 3, 0, 0, -40, 0, }, /* 462 */ - { 28, 12, 3, 0, 0, -129, 0, }, /* 463 */ - { 10, 10, 5, 0, 0, -16, 0, }, /* 464 */ - { 10, 7, 12, 0, 0, 15, 0, }, /* 465 */ - { 10, 7, 12, 0, 0, -16, 0, }, /* 466 */ - { 10, 10, 5, 0, 0, -37, 0, }, /* 467 */ - { 28, 12, 3, 0, 0, -80, 0, }, /* 468 */ - { 10, 10, 5, 0, 0, 3, 0, }, /* 469 */ - { 28, 12, 3, 0, 0, -37, 0, }, /* 470 */ - { 13, 5, 12, 0, 0, 13, 0, }, /* 471 */ - { 13, 6, 12, 0, 0, 13, 0, }, /* 472 */ - { 34, 5, 12, 0, 35332, 34, 0, }, /* 473 */ - { 34, 5, 12, 0, 3814, 34, 0, }, /* 474 */ - { 34, 9, 12, 92, 1, 34, 0, }, /* 475 */ - { 34, 5, 12, 92, -1, 34, 0, }, /* 476 */ - { 34, 5, 12, 92, -58, 34, 0, }, /* 477 */ - { 34, 9, 12, 0, -7615, 34, 0, }, /* 478 */ - { 20, 5, 12, 0, 8, 20, 0, }, /* 479 */ - { 20, 9, 12, 0, -8, 20, 0, }, /* 480 */ - { 20, 5, 12, 0, 74, 20, 0, }, /* 481 */ - { 20, 5, 12, 0, 86, 20, 0, }, /* 482 */ - { 20, 5, 12, 0, 100, 20, 0, }, /* 483 */ - { 20, 5, 12, 0, 128, 20, 0, }, /* 484 */ - { 20, 5, 12, 0, 112, 20, 0, }, /* 485 */ - { 20, 5, 12, 0, 126, 20, 0, }, /* 486 */ - { 20, 8, 12, 0, -8, 20, 0, }, /* 487 */ - { 20, 5, 12, 0, 9, 20, 0, }, /* 488 */ - { 20, 9, 12, 0, -74, 20, 0, }, /* 489 */ - { 20, 8, 12, 0, -9, 20, 0, }, /* 490 */ - { 20, 5, 12, 21, -7173, 20, 0, }, /* 491 */ - { 20, 9, 12, 0, -86, 20, 0, }, /* 492 */ - { 20, 9, 12, 0, -100, 20, 0, }, /* 493 */ - { 20, 9, 12, 0, -112, 20, 0, }, /* 494 */ - { 20, 9, 12, 0, -128, 20, 0, }, /* 495 */ - { 20, 9, 12, 0, -126, 20, 0, }, /* 496 */ - { 28, 1, 3, 0, 0, 28, 0, }, /* 497 */ - { 28, 1, 13, 0, 0, 28, 0, }, /* 498 */ - { 10, 27, 2, 0, 0, 10, 0, }, /* 499 */ - { 10, 28, 2, 0, 0, 10, 0, }, /* 500 */ - { 10, 21, 14, 0, 0, 10, 0, }, /* 501 */ - { 0, 2, 2, 0, 0, 0, 0, }, /* 502 */ - { 28, 12, 3, 0, 0, -84, 0, }, /* 503 */ - { 10, 9, 12, 0, 0, 10, 0, }, /* 504 */ - { 10, 5, 12, 0, 0, 10, 0, }, /* 505 */ - { 20, 9, 12, 96, -7517, 20, 0, }, /* 506 */ - { 34, 9, 12, 100, -8383, 34, 0, }, /* 507 */ - { 34, 9, 12, 104, -8262, 34, 0, }, /* 508 */ - { 34, 9, 12, 0, 28, 34, 0, }, /* 509 */ - { 10, 7, 12, 0, 0, 10, 0, }, /* 510 */ - { 10, 5, 14, 0, 0, 10, 0, }, /* 511 */ - { 34, 5, 12, 0, -28, 34, 0, }, /* 512 */ - { 34, 14, 12, 0, 16, 34, 0, }, /* 513 */ - { 34, 14, 12, 0, -16, 34, 0, }, /* 514 */ - { 34, 14, 12, 0, 0, 34, 0, }, /* 515 */ - { 10, 25, 14, 0, 0, 10, 0, }, /* 516 */ - { 10, 26, 12, 0, 26, 10, 0, }, /* 517 */ - { 10, 26, 14, 0, 26, 10, 0, }, /* 518 */ - { 10, 26, 12, 0, -26, 10, 0, }, /* 519 */ - { 5, 26, 12, 0, 0, 5, 0, }, /* 520 */ - { 18, 9, 12, 0, 48, 18, 0, }, /* 521 */ - { 18, 5, 12, 0, -48, 18, 0, }, /* 522 */ - { 34, 9, 12, 0, -10743, 34, 0, }, /* 523 */ - { 34, 9, 12, 0, -3814, 34, 0, }, /* 524 */ - { 34, 9, 12, 0, -10727, 34, 0, }, /* 525 */ - { 34, 5, 12, 0, -10795, 34, 0, }, /* 526 */ - { 34, 5, 12, 0, -10792, 34, 0, }, /* 527 */ - { 34, 9, 12, 0, -10780, 34, 0, }, /* 528 */ - { 34, 9, 12, 0, -10749, 34, 0, }, /* 529 */ - { 34, 9, 12, 0, -10783, 34, 0, }, /* 530 */ - { 34, 9, 12, 0, -10782, 34, 0, }, /* 531 */ - { 34, 9, 12, 0, -10815, 34, 0, }, /* 532 */ - { 11, 5, 12, 0, 0, 11, 0, }, /* 533 */ - { 11, 26, 12, 0, 0, 11, 0, }, /* 534 */ - { 11, 12, 3, 0, 0, 11, 0, }, /* 535 */ - { 11, 21, 12, 0, 0, 11, 0, }, /* 536 */ - { 11, 15, 12, 0, 0, 11, 0, }, /* 537 */ - { 17, 5, 12, 0, -7264, 17, 0, }, /* 538 */ - { 59, 7, 12, 0, 0, 59, 0, }, /* 539 */ - { 59, 6, 12, 0, 0, 59, 0, }, /* 540 */ - { 59, 21, 12, 0, 0, 59, 0, }, /* 541 */ - { 59, 12, 3, 0, 0, 59, 0, }, /* 542 */ - { 13, 12, 3, 0, 0, 13, 0, }, /* 543 */ - { 10, 21, 12, 0, 0, -28, 0, }, /* 544 */ - { 23, 26, 12, 0, 0, 23, 0, }, /* 545 */ - { 10, 21, 12, 0, 0, -122, 0, }, /* 546 */ - { 10, 21, 12, 0, 0, -116, 0, }, /* 547 */ - { 23, 6, 12, 0, 0, 23, 0, }, /* 548 */ - { 10, 7, 12, 0, 0, 23, 0, }, /* 549 */ - { 23, 14, 12, 0, 0, 23, 0, }, /* 550 */ - { 10, 22, 12, 0, 0, -122, 0, }, /* 551 */ - { 10, 18, 12, 0, 0, -122, 0, }, /* 552 */ - { 10, 26, 12, 0, 0, -116, 0, }, /* 553 */ - { 10, 17, 12, 0, 0, -116, 0, }, /* 554 */ - { 10, 22, 12, 0, 0, -116, 0, }, /* 555 */ - { 10, 18, 12, 0, 0, -116, 0, }, /* 556 */ - { 28, 12, 3, 0, 0, -19, 0, }, /* 557 */ - { 24, 10, 3, 0, 0, 24, 0, }, /* 558 */ - { 10, 17, 14, 0, 0, -116, 0, }, /* 559 */ - { 10, 6, 12, 0, 0, -58, 0, }, /* 560 */ - { 10, 7, 12, 0, 0, -88, 0, }, /* 561 */ - { 10, 21, 14, 0, 0, -88, 0, }, /* 562 */ - { 10, 26, 12, 0, 0, 23, 0, }, /* 563 */ - { 27, 7, 12, 0, 0, 27, 0, }, /* 564 */ - { 28, 12, 3, 0, 0, -58, 0, }, /* 565 */ - { 10, 24, 12, 0, 0, -58, 0, }, /* 566 */ - { 27, 6, 12, 0, 0, 27, 0, }, /* 567 */ - { 10, 17, 12, 0, 0, -58, 0, }, /* 568 */ - { 30, 7, 12, 0, 0, 30, 0, }, /* 569 */ - { 30, 6, 12, 0, 0, 30, 0, }, /* 570 */ - { 4, 7, 12, 0, 0, 4, 0, }, /* 571 */ - { 24, 7, 12, 0, 0, 24, 0, }, /* 572 */ - { 10, 15, 12, 0, 0, 23, 0, }, /* 573 */ - { 24, 26, 12, 0, 0, 24, 0, }, /* 574 */ - { 10, 26, 14, 0, 0, 23, 0, }, /* 575 */ - { 30, 26, 12, 0, 0, 30, 0, }, /* 576 */ - { 23, 7, 12, 0, 0, 23, 0, }, /* 577 */ - { 61, 7, 12, 0, 0, 61, 0, }, /* 578 */ - { 61, 6, 12, 0, 0, 61, 0, }, /* 579 */ - { 61, 26, 12, 0, 0, 61, 0, }, /* 580 */ - { 86, 7, 12, 0, 0, 86, 0, }, /* 581 */ - { 86, 6, 12, 0, 0, 86, 0, }, /* 582 */ - { 86, 21, 12, 0, 0, 86, 0, }, /* 583 */ - { 77, 7, 12, 0, 0, 77, 0, }, /* 584 */ - { 77, 6, 12, 0, 0, 77, 0, }, /* 585 */ - { 77, 21, 12, 0, 0, 77, 0, }, /* 586 */ - { 77, 13, 12, 0, 0, 77, 0, }, /* 587 */ - { 13, 9, 12, 108, 1, 13, 0, }, /* 588 */ - { 13, 5, 12, 108, -35267, 13, 0, }, /* 589 */ - { 13, 7, 12, 0, 0, 13, 0, }, /* 590 */ - { 13, 21, 12, 0, 0, 13, 0, }, /* 591 */ - { 79, 7, 12, 0, 0, 79, 0, }, /* 592 */ - { 79, 14, 12, 0, 0, 79, 0, }, /* 593 */ - { 79, 12, 3, 0, 0, 79, 0, }, /* 594 */ - { 79, 21, 12, 0, 0, 79, 0, }, /* 595 */ - { 34, 9, 12, 0, -35332, 34, 0, }, /* 596 */ - { 34, 9, 12, 0, -42280, 34, 0, }, /* 597 */ - { 34, 9, 12, 0, -42308, 34, 0, }, /* 598 */ - { 34, 9, 12, 0, -42319, 34, 0, }, /* 599 */ - { 34, 9, 12, 0, -42315, 34, 0, }, /* 600 */ - { 34, 9, 12, 0, -42305, 34, 0, }, /* 601 */ - { 34, 9, 12, 0, -42258, 34, 0, }, /* 602 */ - { 34, 9, 12, 0, -42282, 34, 0, }, /* 603 */ - { 34, 9, 12, 0, -42261, 34, 0, }, /* 604 */ - { 34, 9, 12, 0, 928, 34, 0, }, /* 605 */ - { 49, 7, 12, 0, 0, 49, 0, }, /* 606 */ - { 49, 12, 3, 0, 0, 49, 0, }, /* 607 */ - { 49, 10, 5, 0, 0, 49, 0, }, /* 608 */ - { 49, 26, 12, 0, 0, 49, 0, }, /* 609 */ - { 10, 15, 12, 0, 0, -197, 0, }, /* 610 */ - { 10, 15, 12, 0, 0, -170, 0, }, /* 611 */ - { 10, 26, 12, 0, 0, -145, 0, }, /* 612 */ - { 10, 23, 12, 0, 0, -145, 0, }, /* 613 */ - { 65, 7, 12, 0, 0, 65, 0, }, /* 614 */ - { 65, 21, 12, 0, 0, 65, 0, }, /* 615 */ - { 75, 10, 5, 0, 0, 75, 0, }, /* 616 */ - { 75, 7, 12, 0, 0, 75, 0, }, /* 617 */ - { 75, 12, 3, 0, 0, 75, 0, }, /* 618 */ - { 75, 21, 12, 0, 0, 75, 0, }, /* 619 */ - { 75, 13, 12, 0, 0, 75, 0, }, /* 620 */ - { 15, 12, 3, 0, 0, -16, 0, }, /* 621 */ - { 15, 7, 12, 0, 0, -43, 0, }, /* 622 */ - { 69, 13, 12, 0, 0, 69, 0, }, /* 623 */ - { 69, 7, 12, 0, 0, 69, 0, }, /* 624 */ - { 69, 12, 3, 0, 0, 69, 0, }, /* 625 */ - { 10, 21, 12, 0, 0, -92, 0, }, /* 626 */ - { 69, 21, 12, 0, 0, 69, 0, }, /* 627 */ - { 74, 7, 12, 0, 0, 74, 0, }, /* 628 */ - { 74, 12, 3, 0, 0, 74, 0, }, /* 629 */ - { 74, 10, 5, 0, 0, 74, 0, }, /* 630 */ - { 74, 21, 12, 0, 0, 74, 0, }, /* 631 */ - { 84, 12, 3, 0, 0, 84, 0, }, /* 632 */ - { 84, 10, 5, 0, 0, 84, 0, }, /* 633 */ - { 84, 7, 12, 0, 0, 84, 0, }, /* 634 */ - { 84, 21, 12, 0, 0, 84, 0, }, /* 635 */ - { 10, 6, 12, 0, 0, -22, 0, }, /* 636 */ - { 84, 13, 12, 0, 0, 84, 0, }, /* 637 */ - { 39, 6, 12, 0, 0, 39, 0, }, /* 638 */ - { 68, 7, 12, 0, 0, 68, 0, }, /* 639 */ - { 68, 12, 3, 0, 0, 68, 0, }, /* 640 */ - { 68, 10, 5, 0, 0, 68, 0, }, /* 641 */ - { 68, 13, 12, 0, 0, 68, 0, }, /* 642 */ - { 68, 21, 12, 0, 0, 68, 0, }, /* 643 */ - { 92, 7, 12, 0, 0, 92, 0, }, /* 644 */ - { 92, 12, 3, 0, 0, 92, 0, }, /* 645 */ - { 92, 6, 12, 0, 0, 92, 0, }, /* 646 */ - { 92, 21, 12, 0, 0, 92, 0, }, /* 647 */ - { 87, 7, 12, 0, 0, 87, 0, }, /* 648 */ - { 87, 10, 5, 0, 0, 87, 0, }, /* 649 */ - { 87, 12, 3, 0, 0, 87, 0, }, /* 650 */ - { 87, 21, 12, 0, 0, 87, 0, }, /* 651 */ - { 87, 6, 12, 0, 0, 87, 0, }, /* 652 */ - { 34, 5, 12, 0, -928, 34, 0, }, /* 653 */ - { 9, 5, 12, 0, -38864, 9, 0, }, /* 654 */ - { 87, 13, 12, 0, 0, 87, 0, }, /* 655 */ - { 24, 7, 9, 0, 0, 24, 0, }, /* 656 */ - { 24, 7, 10, 0, 0, 24, 0, }, /* 657 */ - { 0, 4, 2, 0, 0, 0, 0, }, /* 658 */ - { 0, 3, 12, 0, 0, 0, 0, }, /* 659 */ - { 26, 25, 12, 0, 0, 26, 0, }, /* 660 */ - { 1, 24, 12, 0, 0, 1, 0, }, /* 661 */ - { 1, 7, 12, 0, 0, -10, 0, }, /* 662 */ - { 1, 26, 12, 0, 0, -10, 0, }, /* 663 */ - { 10, 6, 3, 0, 0, -58, 0, }, /* 664 */ - { 36, 7, 12, 0, 0, 36, 0, }, /* 665 */ - { 10, 21, 12, 0, 0, -25, 0, }, /* 666 */ - { 10, 15, 12, 0, 0, -76, 0, }, /* 667 */ - { 10, 26, 12, 0, 0, -25, 0, }, /* 668 */ - { 20, 14, 12, 0, 0, 20, 0, }, /* 669 */ - { 20, 15, 12, 0, 0, 20, 0, }, /* 670 */ - { 20, 26, 12, 0, 0, 20, 0, }, /* 671 */ - { 71, 7, 12, 0, 0, 71, 0, }, /* 672 */ - { 67, 7, 12, 0, 0, 67, 0, }, /* 673 */ - { 28, 12, 3, 0, 0, -1, 0, }, /* 674 */ - { 10, 15, 12, 0, 0, -1, 0, }, /* 675 */ - { 42, 7, 12, 0, 0, 42, 0, }, /* 676 */ - { 42, 15, 12, 0, 0, 42, 0, }, /* 677 */ - { 19, 7, 12, 0, 0, 19, 0, }, /* 678 */ - { 19, 14, 12, 0, 0, 19, 0, }, /* 679 */ - { 118, 7, 12, 0, 0, 118, 0, }, /* 680 */ - { 118, 12, 3, 0, 0, 118, 0, }, /* 681 */ - { 60, 7, 12, 0, 0, 60, 0, }, /* 682 */ - { 60, 21, 12, 0, 0, 60, 0, }, /* 683 */ - { 43, 7, 12, 0, 0, 43, 0, }, /* 684 */ - { 43, 21, 12, 0, 0, 43, 0, }, /* 685 */ - { 43, 14, 12, 0, 0, 43, 0, }, /* 686 */ - { 14, 9, 12, 0, 40, 14, 0, }, /* 687 */ - { 14, 5, 12, 0, -40, 14, 0, }, /* 688 */ - { 47, 7, 12, 0, 0, 47, 0, }, /* 689 */ - { 45, 7, 12, 0, 0, 45, 0, }, /* 690 */ - { 45, 13, 12, 0, 0, 45, 0, }, /* 691 */ - { 136, 9, 12, 0, 40, 136, 0, }, /* 692 */ - { 136, 5, 12, 0, -40, 136, 0, }, /* 693 */ - { 106, 7, 12, 0, 0, 106, 0, }, /* 694 */ - { 104, 7, 12, 0, 0, 104, 0, }, /* 695 */ - { 104, 21, 12, 0, 0, 104, 0, }, /* 696 */ - { 110, 7, 12, 0, 0, 110, 0, }, /* 697 */ - { 12, 7, 12, 0, 0, 12, 0, }, /* 698 */ - { 81, 7, 12, 0, 0, 81, 0, }, /* 699 */ - { 81, 21, 12, 0, 0, 81, 0, }, /* 700 */ - { 81, 15, 12, 0, 0, 81, 0, }, /* 701 */ - { 120, 7, 12, 0, 0, 120, 0, }, /* 702 */ - { 120, 26, 12, 0, 0, 120, 0, }, /* 703 */ - { 120, 15, 12, 0, 0, 120, 0, }, /* 704 */ - { 116, 7, 12, 0, 0, 116, 0, }, /* 705 */ - { 116, 15, 12, 0, 0, 116, 0, }, /* 706 */ - { 128, 7, 12, 0, 0, 128, 0, }, /* 707 */ - { 128, 15, 12, 0, 0, 128, 0, }, /* 708 */ - { 66, 7, 12, 0, 0, 66, 0, }, /* 709 */ - { 66, 15, 12, 0, 0, 66, 0, }, /* 710 */ - { 66, 21, 12, 0, 0, 66, 0, }, /* 711 */ - { 72, 7, 12, 0, 0, 72, 0, }, /* 712 */ - { 72, 21, 12, 0, 0, 72, 0, }, /* 713 */ - { 98, 7, 12, 0, 0, 98, 0, }, /* 714 */ - { 97, 7, 12, 0, 0, 97, 0, }, /* 715 */ - { 97, 15, 12, 0, 0, 97, 0, }, /* 716 */ - { 31, 7, 12, 0, 0, 31, 0, }, /* 717 */ - { 31, 12, 3, 0, 0, 31, 0, }, /* 718 */ - { 31, 15, 12, 0, 0, 31, 0, }, /* 719 */ - { 31, 21, 12, 0, 0, 31, 0, }, /* 720 */ - { 88, 7, 12, 0, 0, 88, 0, }, /* 721 */ - { 88, 15, 12, 0, 0, 88, 0, }, /* 722 */ - { 88, 21, 12, 0, 0, 88, 0, }, /* 723 */ - { 117, 7, 12, 0, 0, 117, 0, }, /* 724 */ - { 117, 15, 12, 0, 0, 117, 0, }, /* 725 */ - { 112, 7, 12, 0, 0, 112, 0, }, /* 726 */ - { 112, 26, 12, 0, 0, 112, 0, }, /* 727 */ - { 112, 12, 3, 0, 0, 112, 0, }, /* 728 */ - { 112, 15, 12, 0, 0, 112, 0, }, /* 729 */ - { 112, 21, 12, 0, 0, 112, 0, }, /* 730 */ - { 78, 7, 12, 0, 0, 78, 0, }, /* 731 */ - { 78, 21, 12, 0, 0, 78, 0, }, /* 732 */ - { 83, 7, 12, 0, 0, 83, 0, }, /* 733 */ - { 83, 15, 12, 0, 0, 83, 0, }, /* 734 */ - { 82, 7, 12, 0, 0, 82, 0, }, /* 735 */ - { 82, 15, 12, 0, 0, 82, 0, }, /* 736 */ - { 121, 7, 12, 0, 0, 121, 0, }, /* 737 */ - { 121, 21, 12, 0, 0, 121, 0, }, /* 738 */ - { 121, 15, 12, 0, 0, 121, 0, }, /* 739 */ - { 89, 7, 12, 0, 0, 89, 0, }, /* 740 */ - { 130, 9, 12, 0, 64, 130, 0, }, /* 741 */ - { 130, 5, 12, 0, -64, 130, 0, }, /* 742 */ - { 130, 15, 12, 0, 0, 130, 0, }, /* 743 */ - { 144, 7, 12, 0, 0, 144, 0, }, /* 744 */ - { 144, 12, 3, 0, 0, 144, 0, }, /* 745 */ - { 144, 13, 12, 0, 0, 144, 0, }, /* 746 */ - { 1, 15, 12, 0, 0, 1, 0, }, /* 747 */ - { 147, 7, 12, 0, 0, 147, 0, }, /* 748 */ - { 147, 15, 12, 0, 0, 147, 0, }, /* 749 */ - { 148, 7, 12, 0, 0, 148, 0, }, /* 750 */ - { 148, 12, 3, 0, 0, 148, 0, }, /* 751 */ - { 148, 15, 12, 0, 0, 148, 0, }, /* 752 */ - { 148, 21, 12, 0, 0, 148, 0, }, /* 753 */ - { 94, 10, 5, 0, 0, 94, 0, }, /* 754 */ - { 94, 12, 3, 0, 0, 94, 0, }, /* 755 */ - { 94, 7, 12, 0, 0, 94, 0, }, /* 756 */ - { 94, 21, 12, 0, 0, 94, 0, }, /* 757 */ - { 94, 15, 12, 0, 0, 94, 0, }, /* 758 */ - { 94, 13, 12, 0, 0, 94, 0, }, /* 759 */ - { 85, 12, 3, 0, 0, 85, 0, }, /* 760 */ - { 85, 10, 5, 0, 0, 85, 0, }, /* 761 */ - { 85, 7, 12, 0, 0, 85, 0, }, /* 762 */ - { 85, 21, 12, 0, 0, 85, 0, }, /* 763 */ - { 85, 1, 4, 0, 0, 85, 0, }, /* 764 */ - { 101, 7, 12, 0, 0, 101, 0, }, /* 765 */ - { 101, 13, 12, 0, 0, 101, 0, }, /* 766 */ - { 96, 12, 3, 0, 0, 96, 0, }, /* 767 */ - { 96, 7, 12, 0, 0, 96, 0, }, /* 768 */ - { 96, 10, 5, 0, 0, 96, 0, }, /* 769 */ - { 96, 13, 12, 0, 0, 96, 0, }, /* 770 */ - { 96, 21, 12, 0, 0, 96, 0, }, /* 771 */ - { 111, 7, 12, 0, 0, 111, 0, }, /* 772 */ - { 111, 12, 3, 0, 0, 111, 0, }, /* 773 */ - { 111, 21, 12, 0, 0, 111, 0, }, /* 774 */ - { 100, 12, 3, 0, 0, 100, 0, }, /* 775 */ - { 100, 10, 5, 0, 0, 100, 0, }, /* 776 */ - { 100, 7, 12, 0, 0, 100, 0, }, /* 777 */ - { 100, 7, 4, 0, 0, 100, 0, }, /* 778 */ - { 100, 21, 12, 0, 0, 100, 0, }, /* 779 */ - { 100, 13, 12, 0, 0, 100, 0, }, /* 780 */ - { 48, 15, 12, 0, 0, 48, 0, }, /* 781 */ - { 108, 7, 12, 0, 0, 108, 0, }, /* 782 */ - { 108, 10, 5, 0, 0, 108, 0, }, /* 783 */ - { 108, 12, 3, 0, 0, 108, 0, }, /* 784 */ - { 108, 21, 12, 0, 0, 108, 0, }, /* 785 */ - { 129, 7, 12, 0, 0, 129, 0, }, /* 786 */ - { 129, 21, 12, 0, 0, 129, 0, }, /* 787 */ - { 109, 7, 12, 0, 0, 109, 0, }, /* 788 */ - { 109, 12, 3, 0, 0, 109, 0, }, /* 789 */ - { 109, 10, 5, 0, 0, 109, 0, }, /* 790 */ - { 109, 13, 12, 0, 0, 109, 0, }, /* 791 */ - { 107, 12, 3, 0, 0, 107, 0, }, /* 792 */ - { 107, 12, 3, 0, 0, -49, 0, }, /* 793 */ - { 107, 10, 5, 0, 0, 107, 0, }, /* 794 */ - { 107, 10, 5, 0, 0, -49, 0, }, /* 795 */ - { 107, 7, 12, 0, 0, 107, 0, }, /* 796 */ - { 28, 12, 3, 0, 0, -49, 0, }, /* 797 */ - { 107, 10, 3, 0, 0, 107, 0, }, /* 798 */ - { 135, 7, 12, 0, 0, 135, 0, }, /* 799 */ - { 135, 10, 5, 0, 0, 135, 0, }, /* 800 */ - { 135, 12, 3, 0, 0, 135, 0, }, /* 801 */ - { 135, 21, 12, 0, 0, 135, 0, }, /* 802 */ - { 135, 13, 12, 0, 0, 135, 0, }, /* 803 */ - { 124, 7, 12, 0, 0, 124, 0, }, /* 804 */ - { 124, 10, 3, 0, 0, 124, 0, }, /* 805 */ - { 124, 10, 5, 0, 0, 124, 0, }, /* 806 */ - { 124, 12, 3, 0, 0, 124, 0, }, /* 807 */ - { 124, 21, 12, 0, 0, 124, 0, }, /* 808 */ - { 124, 13, 12, 0, 0, 124, 0, }, /* 809 */ - { 123, 7, 12, 0, 0, 123, 0, }, /* 810 */ - { 123, 10, 3, 0, 0, 123, 0, }, /* 811 */ - { 123, 10, 5, 0, 0, 123, 0, }, /* 812 */ - { 123, 12, 3, 0, 0, 123, 0, }, /* 813 */ - { 123, 21, 12, 0, 0, 123, 0, }, /* 814 */ - { 114, 7, 12, 0, 0, 114, 0, }, /* 815 */ - { 114, 10, 5, 0, 0, 114, 0, }, /* 816 */ - { 114, 12, 3, 0, 0, 114, 0, }, /* 817 */ - { 114, 21, 12, 0, 0, 114, 0, }, /* 818 */ - { 114, 13, 12, 0, 0, 114, 0, }, /* 819 */ - { 102, 7, 12, 0, 0, 102, 0, }, /* 820 */ - { 102, 12, 3, 0, 0, 102, 0, }, /* 821 */ - { 102, 10, 5, 0, 0, 102, 0, }, /* 822 */ - { 102, 13, 12, 0, 0, 102, 0, }, /* 823 */ - { 126, 7, 12, 0, 0, 126, 0, }, /* 824 */ - { 126, 12, 3, 0, 0, 126, 0, }, /* 825 */ - { 126, 10, 5, 0, 0, 126, 0, }, /* 826 */ - { 126, 13, 12, 0, 0, 126, 0, }, /* 827 */ - { 126, 15, 12, 0, 0, 126, 0, }, /* 828 */ - { 126, 21, 12, 0, 0, 126, 0, }, /* 829 */ - { 126, 26, 12, 0, 0, 126, 0, }, /* 830 */ - { 142, 7, 12, 0, 0, 142, 0, }, /* 831 */ - { 142, 10, 5, 0, 0, 142, 0, }, /* 832 */ - { 142, 12, 3, 0, 0, 142, 0, }, /* 833 */ - { 142, 21, 12, 0, 0, 142, 0, }, /* 834 */ - { 125, 9, 12, 0, 32, 125, 0, }, /* 835 */ - { 125, 5, 12, 0, -32, 125, 0, }, /* 836 */ - { 125, 13, 12, 0, 0, 125, 0, }, /* 837 */ - { 125, 15, 12, 0, 0, 125, 0, }, /* 838 */ - { 125, 7, 12, 0, 0, 125, 0, }, /* 839 */ - { 141, 7, 12, 0, 0, 141, 0, }, /* 840 */ - { 141, 12, 3, 0, 0, 141, 0, }, /* 841 */ - { 141, 10, 5, 0, 0, 141, 0, }, /* 842 */ - { 141, 7, 4, 0, 0, 141, 0, }, /* 843 */ - { 141, 21, 12, 0, 0, 141, 0, }, /* 844 */ - { 140, 7, 12, 0, 0, 140, 0, }, /* 845 */ - { 140, 12, 3, 0, 0, 140, 0, }, /* 846 */ - { 140, 10, 5, 0, 0, 140, 0, }, /* 847 */ - { 140, 7, 4, 0, 0, 140, 0, }, /* 848 */ - { 140, 21, 12, 0, 0, 140, 0, }, /* 849 */ - { 122, 7, 12, 0, 0, 122, 0, }, /* 850 */ - { 133, 7, 12, 0, 0, 133, 0, }, /* 851 */ - { 133, 10, 5, 0, 0, 133, 0, }, /* 852 */ - { 133, 12, 3, 0, 0, 133, 0, }, /* 853 */ - { 133, 21, 12, 0, 0, 133, 0, }, /* 854 */ - { 133, 13, 12, 0, 0, 133, 0, }, /* 855 */ - { 133, 15, 12, 0, 0, 133, 0, }, /* 856 */ - { 134, 21, 12, 0, 0, 134, 0, }, /* 857 */ - { 134, 7, 12, 0, 0, 134, 0, }, /* 858 */ - { 134, 12, 3, 0, 0, 134, 0, }, /* 859 */ - { 134, 10, 5, 0, 0, 134, 0, }, /* 860 */ - { 138, 7, 12, 0, 0, 138, 0, }, /* 861 */ - { 138, 12, 3, 0, 0, 138, 0, }, /* 862 */ - { 138, 7, 4, 0, 0, 138, 0, }, /* 863 */ - { 138, 13, 12, 0, 0, 138, 0, }, /* 864 */ - { 143, 7, 12, 0, 0, 143, 0, }, /* 865 */ - { 143, 10, 5, 0, 0, 143, 0, }, /* 866 */ - { 143, 12, 3, 0, 0, 143, 0, }, /* 867 */ - { 143, 13, 12, 0, 0, 143, 0, }, /* 868 */ - { 145, 7, 12, 0, 0, 145, 0, }, /* 869 */ - { 145, 12, 3, 0, 0, 145, 0, }, /* 870 */ - { 145, 10, 5, 0, 0, 145, 0, }, /* 871 */ - { 145, 21, 12, 0, 0, 145, 0, }, /* 872 */ - { 63, 7, 12, 0, 0, 63, 0, }, /* 873 */ - { 63, 14, 12, 0, 0, 63, 0, }, /* 874 */ - { 63, 21, 12, 0, 0, 63, 0, }, /* 875 */ - { 80, 7, 12, 0, 0, 80, 0, }, /* 876 */ - { 127, 7, 12, 0, 0, 127, 0, }, /* 877 */ - { 115, 7, 12, 0, 0, 115, 0, }, /* 878 */ - { 115, 13, 12, 0, 0, 115, 0, }, /* 879 */ - { 115, 21, 12, 0, 0, 115, 0, }, /* 880 */ - { 103, 7, 12, 0, 0, 103, 0, }, /* 881 */ - { 103, 12, 3, 0, 0, 103, 0, }, /* 882 */ - { 103, 21, 12, 0, 0, 103, 0, }, /* 883 */ - { 119, 7, 12, 0, 0, 119, 0, }, /* 884 */ - { 119, 12, 3, 0, 0, 119, 0, }, /* 885 */ - { 119, 21, 12, 0, 0, 119, 0, }, /* 886 */ - { 119, 26, 12, 0, 0, 119, 0, }, /* 887 */ - { 119, 6, 12, 0, 0, 119, 0, }, /* 888 */ - { 119, 13, 12, 0, 0, 119, 0, }, /* 889 */ - { 119, 15, 12, 0, 0, 119, 0, }, /* 890 */ - { 146, 9, 12, 0, 32, 146, 0, }, /* 891 */ - { 146, 5, 12, 0, -32, 146, 0, }, /* 892 */ - { 146, 15, 12, 0, 0, 146, 0, }, /* 893 */ - { 146, 21, 12, 0, 0, 146, 0, }, /* 894 */ - { 99, 7, 12, 0, 0, 99, 0, }, /* 895 */ - { 99, 10, 5, 0, 0, 99, 0, }, /* 896 */ - { 99, 12, 3, 0, 0, 99, 0, }, /* 897 */ - { 99, 6, 12, 0, 0, 99, 0, }, /* 898 */ - { 137, 6, 12, 0, 0, 137, 0, }, /* 899 */ - { 139, 6, 12, 0, 0, 139, 0, }, /* 900 */ - { 137, 7, 12, 0, 0, 137, 0, }, /* 901 */ - { 139, 7, 12, 0, 0, 139, 0, }, /* 902 */ - { 105, 7, 12, 0, 0, 105, 0, }, /* 903 */ - { 105, 26, 12, 0, 0, 105, 0, }, /* 904 */ - { 105, 12, 3, 0, 0, 105, 0, }, /* 905 */ - { 105, 21, 12, 0, 0, 105, 0, }, /* 906 */ - { 10, 1, 2, 0, 0, 105, 0, }, /* 907 */ - { 10, 10, 3, 0, 0, 10, 0, }, /* 908 */ - { 10, 10, 5, 0, 0, 10, 0, }, /* 909 */ - { 20, 12, 3, 0, 0, 20, 0, }, /* 910 */ - { 131, 26, 12, 0, 0, 131, 0, }, /* 911 */ - { 131, 12, 3, 0, 0, 131, 0, }, /* 912 */ - { 131, 21, 12, 0, 0, 131, 0, }, /* 913 */ - { 18, 12, 3, 0, 0, 18, 0, }, /* 914 */ - { 113, 7, 12, 0, 0, 113, 0, }, /* 915 */ - { 113, 15, 12, 0, 0, 113, 0, }, /* 916 */ - { 113, 12, 3, 0, 0, 113, 0, }, /* 917 */ - { 132, 9, 12, 0, 34, 132, 0, }, /* 918 */ - { 132, 5, 12, 0, -34, 132, 0, }, /* 919 */ - { 132, 12, 3, 0, 0, 132, 0, }, /* 920 */ - { 132, 13, 12, 0, 0, 132, 0, }, /* 921 */ - { 132, 21, 12, 0, 0, 132, 0, }, /* 922 */ - { 0, 2, 14, 0, 0, 0, 0, }, /* 923 */ - { 10, 26, 11, 0, 0, 10, 0, }, /* 924 */ - { 27, 26, 12, 0, 0, 27, 0, }, /* 925 */ - { 10, 24, 3, 0, 0, 10, 0, }, /* 926 */ - { 10, 1, 3, 0, 0, 10, 0, }, /* 927 */ + { 55, 21, 12, 0, 0, 55, 0, }, /* 301 */ + { 55, 15, 12, 0, 0, 55, 0, }, /* 302 */ + { 55, 26, 12, 0, 0, 55, 0, }, /* 303 */ + { 29, 7, 12, 0, 0, 29, 0, }, /* 304 */ + { 29, 12, 3, 0, 0, 29, 0, }, /* 305 */ + { 29, 10, 5, 0, 0, 29, 0, }, /* 306 */ + { 29, 21, 12, 0, 0, 29, 0, }, /* 307 */ + { 29, 10, 3, 0, 0, 29, 0, }, /* 308 */ + { 29, 13, 12, 0, 0, -67, 0, }, /* 309 */ + { 37, 12, 3, 0, 0, 37, 0, }, /* 310 */ + { 37, 10, 5, 0, 0, 37, 0, }, /* 311 */ + { 37, 7, 12, 0, 0, 37, 0, }, /* 312 */ + { 37, 10, 3, 0, 0, 37, 0, }, /* 313 */ + { 37, 7, 4, 0, 0, 37, 0, }, /* 314 */ + { 37, 26, 12, 0, 0, 37, 0, }, /* 315 */ + { 37, 15, 12, 0, 0, 37, 0, }, /* 316 */ + { 37, 13, 12, 0, 0, 37, 0, }, /* 317 */ + { 48, 12, 3, 0, 0, 48, 0, }, /* 318 */ + { 48, 10, 5, 0, 0, 48, 0, }, /* 319 */ + { 48, 7, 12, 0, 0, 48, 0, }, /* 320 */ + { 48, 10, 3, 0, 0, 48, 0, }, /* 321 */ + { 48, 13, 12, 0, 0, 48, 0, }, /* 322 */ + { 48, 21, 12, 0, 0, 48, 0, }, /* 323 */ + { 57, 7, 12, 0, 0, 57, 0, }, /* 324 */ + { 57, 12, 3, 0, 0, 57, 0, }, /* 325 */ + { 57, 7, 5, 0, 0, 57, 0, }, /* 326 */ + { 57, 6, 12, 0, 0, 57, 0, }, /* 327 */ + { 57, 21, 12, 0, 0, 57, 0, }, /* 328 */ + { 57, 13, 12, 0, 0, 57, 0, }, /* 329 */ + { 33, 7, 12, 0, 0, 33, 0, }, /* 330 */ + { 33, 12, 3, 0, 0, 33, 0, }, /* 331 */ + { 33, 7, 5, 0, 0, 33, 0, }, /* 332 */ + { 33, 6, 12, 0, 0, 33, 0, }, /* 333 */ + { 33, 13, 12, 0, 0, 33, 0, }, /* 334 */ + { 58, 7, 12, 0, 0, 58, 0, }, /* 335 */ + { 58, 26, 12, 0, 0, 58, 0, }, /* 336 */ + { 58, 21, 12, 0, 0, 58, 0, }, /* 337 */ + { 58, 12, 3, 0, 0, 58, 0, }, /* 338 */ + { 58, 13, 12, 0, 0, 58, 0, }, /* 339 */ + { 58, 15, 12, 0, 0, 58, 0, }, /* 340 */ + { 58, 22, 12, 0, 0, 58, 0, }, /* 341 */ + { 58, 18, 12, 0, 0, 58, 0, }, /* 342 */ + { 58, 10, 5, 0, 0, 58, 0, }, /* 343 */ + { 39, 7, 12, 0, 0, 39, 0, }, /* 344 */ + { 39, 10, 12, 0, 0, 39, 0, }, /* 345 */ + { 39, 12, 3, 0, 0, 39, 0, }, /* 346 */ + { 39, 10, 5, 0, 0, 39, 0, }, /* 347 */ + { 39, 13, 12, 0, 0, -88, 0, }, /* 348 */ + { 39, 21, 12, 0, 0, 39, 0, }, /* 349 */ + { 39, 13, 12, 0, 0, 39, 0, }, /* 350 */ + { 39, 26, 12, 0, 0, 39, 0, }, /* 351 */ + { 17, 9, 12, 0, 7264, 17, 0, }, /* 352 */ + { 17, 5, 12, 0, 3008, 17, 0, }, /* 353 */ + { 10, 21, 12, 0, 0, -49, 0, }, /* 354 */ + { 17, 6, 12, 0, 0, 17, 0, }, /* 355 */ + { 24, 7, 6, 0, 0, 24, 0, }, /* 356 */ + { 24, 7, 7, 0, 0, 24, 0, }, /* 357 */ + { 24, 7, 8, 0, 0, 24, 0, }, /* 358 */ + { 16, 7, 12, 0, 0, 16, 0, }, /* 359 */ + { 16, 12, 3, 0, 0, 16, 0, }, /* 360 */ + { 16, 21, 12, 0, 0, 16, 0, }, /* 361 */ + { 16, 15, 12, 0, 0, 16, 0, }, /* 362 */ + { 16, 26, 12, 0, 0, 16, 0, }, /* 363 */ + { 9, 9, 12, 0, 38864, 9, 0, }, /* 364 */ + { 9, 9, 12, 0, 8, 9, 0, }, /* 365 */ + { 9, 5, 12, 0, -8, 9, 0, }, /* 366 */ + { 8, 17, 12, 0, 0, 8, 0, }, /* 367 */ + { 8, 7, 12, 0, 0, 8, 0, }, /* 368 */ + { 8, 26, 12, 0, 0, 8, 0, }, /* 369 */ + { 8, 21, 12, 0, 0, 8, 0, }, /* 370 */ + { 41, 29, 12, 0, 0, 41, 0, }, /* 371 */ + { 41, 7, 12, 0, 0, 41, 0, }, /* 372 */ + { 41, 22, 12, 0, 0, 41, 0, }, /* 373 */ + { 41, 18, 12, 0, 0, 41, 0, }, /* 374 */ + { 46, 7, 12, 0, 0, 46, 0, }, /* 375 */ + { 46, 14, 12, 0, 0, 46, 0, }, /* 376 */ + { 51, 7, 12, 0, 0, 51, 0, }, /* 377 */ + { 51, 12, 3, 0, 0, 51, 0, }, /* 378 */ + { 25, 7, 12, 0, 0, 25, 0, }, /* 379 */ + { 25, 12, 3, 0, 0, 25, 0, }, /* 380 */ + { 10, 21, 12, 0, 0, -117, 0, }, /* 381 */ + { 7, 7, 12, 0, 0, 7, 0, }, /* 382 */ + { 7, 12, 3, 0, 0, 7, 0, }, /* 383 */ + { 52, 7, 12, 0, 0, 52, 0, }, /* 384 */ + { 52, 12, 3, 0, 0, 52, 0, }, /* 385 */ + { 32, 7, 12, 0, 0, 32, 0, }, /* 386 */ + { 32, 12, 3, 0, 0, 32, 0, }, /* 387 */ + { 32, 10, 5, 0, 0, 32, 0, }, /* 388 */ + { 32, 21, 12, 0, 0, 32, 0, }, /* 389 */ + { 32, 6, 12, 0, 0, 32, 0, }, /* 390 */ + { 32, 23, 12, 0, 0, 32, 0, }, /* 391 */ + { 32, 13, 12, 0, 0, 32, 0, }, /* 392 */ + { 32, 15, 12, 0, 0, 32, 0, }, /* 393 */ + { 38, 21, 12, 0, 0, 38, 0, }, /* 394 */ + { 10, 21, 12, 0, 0, -73, 0, }, /* 395 */ + { 38, 17, 12, 0, 0, 38, 0, }, /* 396 */ + { 38, 12, 3, 0, 0, 38, 0, }, /* 397 */ + { 38, 1, 2, 0, 0, 38, 0, }, /* 398 */ + { 38, 13, 12, 0, 0, 38, 0, }, /* 399 */ + { 38, 7, 12, 0, 0, 38, 0, }, /* 400 */ + { 38, 6, 12, 0, 0, 38, 0, }, /* 401 */ + { 35, 7, 12, 0, 0, 35, 0, }, /* 402 */ + { 35, 12, 3, 0, 0, 35, 0, }, /* 403 */ + { 35, 10, 5, 0, 0, 35, 0, }, /* 404 */ + { 35, 26, 12, 0, 0, 35, 0, }, /* 405 */ + { 35, 21, 12, 0, 0, 35, 0, }, /* 406 */ + { 35, 13, 12, 0, 0, 35, 0, }, /* 407 */ + { 53, 7, 12, 0, 0, 53, 0, }, /* 408 */ + { 40, 7, 12, 0, 0, 40, 0, }, /* 409 */ + { 40, 13, 12, 0, 0, 40, 0, }, /* 410 */ + { 40, 15, 12, 0, 0, 40, 0, }, /* 411 */ + { 40, 26, 12, 0, 0, 40, 0, }, /* 412 */ + { 32, 26, 12, 0, 0, 32, 0, }, /* 413 */ + { 6, 7, 12, 0, 0, 6, 0, }, /* 414 */ + { 6, 12, 3, 0, 0, 6, 0, }, /* 415 */ + { 6, 10, 5, 0, 0, 6, 0, }, /* 416 */ + { 6, 21, 12, 0, 0, 6, 0, }, /* 417 */ + { 91, 7, 12, 0, 0, 91, 0, }, /* 418 */ + { 91, 10, 5, 0, 0, 91, 0, }, /* 419 */ + { 91, 12, 3, 0, 0, 91, 0, }, /* 420 */ + { 91, 10, 12, 0, 0, 91, 0, }, /* 421 */ + { 91, 13, 12, 0, 0, 91, 0, }, /* 422 */ + { 91, 21, 12, 0, 0, 91, 0, }, /* 423 */ + { 91, 6, 12, 0, 0, 91, 0, }, /* 424 */ + { 28, 11, 3, 0, 0, 28, 0, }, /* 425 */ + { 62, 12, 3, 0, 0, 62, 0, }, /* 426 */ + { 62, 10, 5, 0, 0, 62, 0, }, /* 427 */ + { 62, 7, 12, 0, 0, 62, 0, }, /* 428 */ + { 62, 10, 3, 0, 0, 62, 0, }, /* 429 */ + { 62, 13, 12, 0, 0, 62, 0, }, /* 430 */ + { 62, 21, 12, 0, 0, 62, 0, }, /* 431 */ + { 62, 26, 12, 0, 0, 62, 0, }, /* 432 */ + { 76, 12, 3, 0, 0, 76, 0, }, /* 433 */ + { 76, 10, 5, 0, 0, 76, 0, }, /* 434 */ + { 76, 7, 12, 0, 0, 76, 0, }, /* 435 */ + { 76, 13, 12, 0, 0, 76, 0, }, /* 436 */ + { 93, 7, 12, 0, 0, 93, 0, }, /* 437 */ + { 93, 12, 3, 0, 0, 93, 0, }, /* 438 */ + { 93, 10, 5, 0, 0, 93, 0, }, /* 439 */ + { 93, 21, 12, 0, 0, 93, 0, }, /* 440 */ + { 70, 7, 12, 0, 0, 70, 0, }, /* 441 */ + { 70, 10, 5, 0, 0, 70, 0, }, /* 442 */ + { 70, 12, 3, 0, 0, 70, 0, }, /* 443 */ + { 70, 21, 12, 0, 0, 70, 0, }, /* 444 */ + { 70, 13, 12, 0, 0, 70, 0, }, /* 445 */ + { 73, 13, 12, 0, 0, 73, 0, }, /* 446 */ + { 73, 7, 12, 0, 0, 73, 0, }, /* 447 */ + { 73, 6, 12, 0, 0, 73, 0, }, /* 448 */ + { 73, 21, 12, 0, 0, 73, 0, }, /* 449 */ + { 13, 5, 12, 63, -6222, 13, 0, }, /* 450 */ + { 13, 5, 12, 67, -6221, 13, 0, }, /* 451 */ + { 13, 5, 12, 71, -6212, 13, 0, }, /* 452 */ + { 13, 5, 12, 75, -6210, 13, 0, }, /* 453 */ + { 13, 5, 12, 79, -6210, 13, 0, }, /* 454 */ + { 13, 5, 12, 79, -6211, 13, 0, }, /* 455 */ + { 13, 5, 12, 84, -6204, 13, 0, }, /* 456 */ + { 13, 5, 12, 88, -6180, 13, 0, }, /* 457 */ + { 13, 5, 12, 108, 35267, 13, 0, }, /* 458 */ + { 17, 9, 12, 0, -3008, 17, 0, }, /* 459 */ + { 76, 21, 12, 0, 0, 76, 0, }, /* 460 */ + { 28, 12, 3, 0, 0, -112, 0, }, /* 461 */ + { 28, 12, 3, 0, 0, 15, 0, }, /* 462 */ + { 10, 21, 12, 0, 0, -37, 0, }, /* 463 */ + { 28, 12, 3, 0, 0, -13, 0, }, /* 464 */ + { 28, 12, 3, 0, 0, -43, 0, }, /* 465 */ + { 28, 12, 3, 0, 0, -146, 0, }, /* 466 */ + { 10, 10, 5, 0, 0, -13, 0, }, /* 467 */ + { 10, 7, 12, 0, 0, -40, 0, }, /* 468 */ + { 10, 7, 12, 0, 0, -13, 0, }, /* 469 */ + { 10, 7, 12, 0, 0, 15, 0, }, /* 470 */ + { 10, 7, 12, 0, 0, -162, 0, }, /* 471 */ + { 10, 7, 12, 0, 0, -37, 0, }, /* 472 */ + { 28, 12, 3, 0, 0, -96, 0, }, /* 473 */ + { 10, 10, 5, 0, 0, 3, 0, }, /* 474 */ + { 28, 12, 3, 0, 0, -37, 0, }, /* 475 */ + { 10, 7, 12, 0, 0, 150, 0, }, /* 476 */ + { 13, 5, 12, 0, 0, 13, 0, }, /* 477 */ + { 13, 6, 12, 0, 0, 13, 0, }, /* 478 */ + { 34, 5, 12, 0, 35332, 34, 0, }, /* 479 */ + { 34, 5, 12, 0, 3814, 34, 0, }, /* 480 */ + { 34, 5, 12, 0, 35384, 34, 0, }, /* 481 */ + { 28, 12, 3, 0, 0, -34, 0, }, /* 482 */ + { 34, 9, 12, 92, 1, 34, 0, }, /* 483 */ + { 34, 5, 12, 92, -1, 34, 0, }, /* 484 */ + { 34, 5, 12, 92, -58, 34, 0, }, /* 485 */ + { 34, 9, 12, 0, -7615, 34, 0, }, /* 486 */ + { 20, 5, 12, 0, 8, 20, 0, }, /* 487 */ + { 20, 9, 12, 0, -8, 20, 0, }, /* 488 */ + { 20, 5, 12, 0, 74, 20, 0, }, /* 489 */ + { 20, 5, 12, 0, 86, 20, 0, }, /* 490 */ + { 20, 5, 12, 0, 100, 20, 0, }, /* 491 */ + { 20, 5, 12, 0, 128, 20, 0, }, /* 492 */ + { 20, 5, 12, 0, 112, 20, 0, }, /* 493 */ + { 20, 5, 12, 0, 126, 20, 0, }, /* 494 */ + { 20, 8, 12, 0, -8, 20, 0, }, /* 495 */ + { 20, 5, 12, 0, 9, 20, 0, }, /* 496 */ + { 20, 9, 12, 0, -74, 20, 0, }, /* 497 */ + { 20, 8, 12, 0, -9, 20, 0, }, /* 498 */ + { 20, 5, 12, 21, -7173, 20, 0, }, /* 499 */ + { 20, 9, 12, 0, -86, 20, 0, }, /* 500 */ + { 20, 9, 12, 0, -100, 20, 0, }, /* 501 */ + { 20, 9, 12, 0, -112, 20, 0, }, /* 502 */ + { 20, 9, 12, 0, -128, 20, 0, }, /* 503 */ + { 20, 9, 12, 0, -126, 20, 0, }, /* 504 */ + { 28, 1, 3, 0, 0, 28, 0, }, /* 505 */ + { 28, 1, 13, 0, 0, 28, 0, }, /* 506 */ + { 10, 27, 2, 0, 0, 10, 0, }, /* 507 */ + { 10, 28, 2, 0, 0, 10, 0, }, /* 508 */ + { 10, 29, 12, 0, 0, -70, 0, }, /* 509 */ + { 10, 21, 14, 0, 0, 10, 0, }, /* 510 */ + { 0, 2, 2, 0, 0, 0, 0, }, /* 511 */ + { 28, 12, 3, 0, 0, -100, 0, }, /* 512 */ + { 10, 9, 12, 0, 0, 10, 0, }, /* 513 */ + { 10, 5, 12, 0, 0, 10, 0, }, /* 514 */ + { 20, 9, 12, 96, -7517, 20, 0, }, /* 515 */ + { 34, 9, 12, 100, -8383, 34, 0, }, /* 516 */ + { 34, 9, 12, 104, -8262, 34, 0, }, /* 517 */ + { 34, 9, 12, 0, 28, 34, 0, }, /* 518 */ + { 10, 7, 12, 0, 0, 10, 0, }, /* 519 */ + { 10, 5, 14, 0, 0, 10, 0, }, /* 520 */ + { 34, 5, 12, 0, -28, 34, 0, }, /* 521 */ + { 34, 14, 12, 0, 16, 34, 0, }, /* 522 */ + { 34, 14, 12, 0, -16, 34, 0, }, /* 523 */ + { 34, 14, 12, 0, 0, 34, 0, }, /* 524 */ + { 10, 25, 14, 0, 0, 10, 0, }, /* 525 */ + { 10, 26, 12, 0, 26, 10, 0, }, /* 526 */ + { 10, 26, 14, 0, 26, 10, 0, }, /* 527 */ + { 10, 26, 12, 0, -26, 10, 0, }, /* 528 */ + { 5, 26, 12, 0, 0, 5, 0, }, /* 529 */ + { 18, 9, 12, 0, 48, 18, 0, }, /* 530 */ + { 18, 5, 12, 0, -48, 18, 0, }, /* 531 */ + { 34, 9, 12, 0, -10743, 34, 0, }, /* 532 */ + { 34, 9, 12, 0, -3814, 34, 0, }, /* 533 */ + { 34, 9, 12, 0, -10727, 34, 0, }, /* 534 */ + { 34, 5, 12, 0, -10795, 34, 0, }, /* 535 */ + { 34, 5, 12, 0, -10792, 34, 0, }, /* 536 */ + { 34, 9, 12, 0, -10780, 34, 0, }, /* 537 */ + { 34, 9, 12, 0, -10749, 34, 0, }, /* 538 */ + { 34, 9, 12, 0, -10783, 34, 0, }, /* 539 */ + { 34, 9, 12, 0, -10782, 34, 0, }, /* 540 */ + { 34, 9, 12, 0, -10815, 34, 0, }, /* 541 */ + { 11, 5, 12, 0, 0, 11, 0, }, /* 542 */ + { 11, 26, 12, 0, 0, 11, 0, }, /* 543 */ + { 11, 12, 3, 0, 0, 11, 0, }, /* 544 */ + { 11, 21, 12, 0, 0, 11, 0, }, /* 545 */ + { 11, 15, 12, 0, 0, 11, 0, }, /* 546 */ + { 17, 5, 12, 0, -7264, 17, 0, }, /* 547 */ + { 59, 7, 12, 0, 0, 59, 0, }, /* 548 */ + { 59, 6, 12, 0, 0, 59, 0, }, /* 549 */ + { 59, 21, 12, 0, 0, 59, 0, }, /* 550 */ + { 59, 12, 3, 0, 0, 59, 0, }, /* 551 */ + { 13, 12, 3, 0, 0, 13, 0, }, /* 552 */ + { 10, 21, 12, 0, 0, -25, 0, }, /* 553 */ + { 23, 26, 12, 0, 0, 23, 0, }, /* 554 */ + { 10, 21, 12, 0, 0, -139, 0, }, /* 555 */ + { 10, 21, 12, 0, 0, -133, 0, }, /* 556 */ + { 23, 6, 12, 0, 0, 23, 0, }, /* 557 */ + { 10, 7, 12, 0, 0, 23, 0, }, /* 558 */ + { 23, 14, 12, 0, 0, 23, 0, }, /* 559 */ + { 10, 22, 12, 0, 0, -139, 0, }, /* 560 */ + { 10, 18, 12, 0, 0, -139, 0, }, /* 561 */ + { 10, 26, 12, 0, 0, -133, 0, }, /* 562 */ + { 10, 17, 12, 0, 0, -133, 0, }, /* 563 */ + { 10, 22, 12, 0, 0, -133, 0, }, /* 564 */ + { 10, 18, 12, 0, 0, -133, 0, }, /* 565 */ + { 28, 12, 3, 0, 0, -16, 0, }, /* 566 */ + { 24, 10, 3, 0, 0, 24, 0, }, /* 567 */ + { 10, 17, 14, 0, 0, -133, 0, }, /* 568 */ + { 10, 6, 12, 0, 0, -64, 0, }, /* 569 */ + { 10, 7, 12, 0, 0, -104, 0, }, /* 570 */ + { 10, 21, 14, 0, 0, -104, 0, }, /* 571 */ + { 10, 26, 12, 0, 0, 23, 0, }, /* 572 */ + { 27, 7, 12, 0, 0, 27, 0, }, /* 573 */ + { 28, 12, 3, 0, 0, -64, 0, }, /* 574 */ + { 10, 24, 12, 0, 0, -64, 0, }, /* 575 */ + { 27, 6, 12, 0, 0, 27, 0, }, /* 576 */ + { 10, 17, 12, 0, 0, -64, 0, }, /* 577 */ + { 30, 7, 12, 0, 0, 30, 0, }, /* 578 */ + { 30, 6, 12, 0, 0, 30, 0, }, /* 579 */ + { 4, 7, 12, 0, 0, 4, 0, }, /* 580 */ + { 24, 7, 12, 0, 0, 24, 0, }, /* 581 */ + { 10, 15, 12, 0, 0, 23, 0, }, /* 582 */ + { 24, 26, 12, 0, 0, 24, 0, }, /* 583 */ + { 10, 26, 14, 0, 0, 23, 0, }, /* 584 */ + { 30, 26, 12, 0, 0, 30, 0, }, /* 585 */ + { 23, 7, 12, 0, 0, 23, 0, }, /* 586 */ + { 61, 7, 12, 0, 0, 61, 0, }, /* 587 */ + { 61, 6, 12, 0, 0, 61, 0, }, /* 588 */ + { 61, 26, 12, 0, 0, 61, 0, }, /* 589 */ + { 86, 7, 12, 0, 0, 86, 0, }, /* 590 */ + { 86, 6, 12, 0, 0, 86, 0, }, /* 591 */ + { 86, 21, 12, 0, 0, 86, 0, }, /* 592 */ + { 77, 7, 12, 0, 0, 77, 0, }, /* 593 */ + { 77, 6, 12, 0, 0, 77, 0, }, /* 594 */ + { 77, 21, 12, 0, 0, 77, 0, }, /* 595 */ + { 77, 13, 12, 0, 0, 77, 0, }, /* 596 */ + { 13, 9, 12, 108, 1, 13, 0, }, /* 597 */ + { 13, 5, 12, 108, -35267, 13, 0, }, /* 598 */ + { 13, 7, 12, 0, 0, 13, 0, }, /* 599 */ + { 13, 21, 12, 0, 0, 13, 0, }, /* 600 */ + { 79, 7, 12, 0, 0, 79, 0, }, /* 601 */ + { 79, 14, 12, 0, 0, 79, 0, }, /* 602 */ + { 79, 12, 3, 0, 0, 79, 0, }, /* 603 */ + { 79, 21, 12, 0, 0, 79, 0, }, /* 604 */ + { 10, 24, 12, 0, 0, -61, 0, }, /* 605 */ + { 34, 9, 12, 0, -35332, 34, 0, }, /* 606 */ + { 34, 9, 12, 0, -42280, 34, 0, }, /* 607 */ + { 34, 5, 12, 0, 48, 34, 0, }, /* 608 */ + { 34, 9, 12, 0, -42308, 34, 0, }, /* 609 */ + { 34, 9, 12, 0, -42319, 34, 0, }, /* 610 */ + { 34, 9, 12, 0, -42315, 34, 0, }, /* 611 */ + { 34, 9, 12, 0, -42305, 34, 0, }, /* 612 */ + { 34, 9, 12, 0, -42258, 34, 0, }, /* 613 */ + { 34, 9, 12, 0, -42282, 34, 0, }, /* 614 */ + { 34, 9, 12, 0, -42261, 34, 0, }, /* 615 */ + { 34, 9, 12, 0, 928, 34, 0, }, /* 616 */ + { 34, 9, 12, 0, -48, 34, 0, }, /* 617 */ + { 34, 9, 12, 0, -42307, 34, 0, }, /* 618 */ + { 34, 9, 12, 0, -35384, 34, 0, }, /* 619 */ + { 49, 7, 12, 0, 0, 49, 0, }, /* 620 */ + { 49, 12, 3, 0, 0, 49, 0, }, /* 621 */ + { 49, 10, 5, 0, 0, 49, 0, }, /* 622 */ + { 49, 26, 12, 0, 0, 49, 0, }, /* 623 */ + { 10, 15, 12, 0, 0, -224, 0, }, /* 624 */ + { 10, 15, 12, 0, 0, -210, 0, }, /* 625 */ + { 10, 26, 12, 0, 0, -171, 0, }, /* 626 */ + { 10, 23, 12, 0, 0, -171, 0, }, /* 627 */ + { 65, 7, 12, 0, 0, 65, 0, }, /* 628 */ + { 65, 21, 12, 0, 0, 65, 0, }, /* 629 */ + { 75, 10, 5, 0, 0, 75, 0, }, /* 630 */ + { 75, 7, 12, 0, 0, 75, 0, }, /* 631 */ + { 75, 12, 3, 0, 0, 75, 0, }, /* 632 */ + { 75, 21, 12, 0, 0, 75, 0, }, /* 633 */ + { 75, 13, 12, 0, 0, 75, 0, }, /* 634 */ + { 15, 12, 3, 0, 0, -13, 0, }, /* 635 */ + { 15, 7, 12, 0, 0, -46, 0, }, /* 636 */ + { 69, 13, 12, 0, 0, 69, 0, }, /* 637 */ + { 69, 7, 12, 0, 0, 69, 0, }, /* 638 */ + { 69, 12, 3, 0, 0, 69, 0, }, /* 639 */ + { 10, 21, 12, 0, 0, -108, 0, }, /* 640 */ + { 69, 21, 12, 0, 0, 69, 0, }, /* 641 */ + { 74, 7, 12, 0, 0, 74, 0, }, /* 642 */ + { 74, 12, 3, 0, 0, 74, 0, }, /* 643 */ + { 74, 10, 5, 0, 0, 74, 0, }, /* 644 */ + { 74, 21, 12, 0, 0, 74, 0, }, /* 645 */ + { 84, 12, 3, 0, 0, 84, 0, }, /* 646 */ + { 84, 10, 5, 0, 0, 84, 0, }, /* 647 */ + { 84, 7, 12, 0, 0, 84, 0, }, /* 648 */ + { 84, 21, 12, 0, 0, 84, 0, }, /* 649 */ + { 10, 6, 12, 0, 0, -19, 0, }, /* 650 */ + { 84, 13, 12, 0, 0, 84, 0, }, /* 651 */ + { 39, 6, 12, 0, 0, 39, 0, }, /* 652 */ + { 68, 7, 12, 0, 0, 68, 0, }, /* 653 */ + { 68, 12, 3, 0, 0, 68, 0, }, /* 654 */ + { 68, 10, 5, 0, 0, 68, 0, }, /* 655 */ + { 68, 13, 12, 0, 0, 68, 0, }, /* 656 */ + { 68, 21, 12, 0, 0, 68, 0, }, /* 657 */ + { 92, 7, 12, 0, 0, 92, 0, }, /* 658 */ + { 92, 12, 3, 0, 0, 92, 0, }, /* 659 */ + { 92, 6, 12, 0, 0, 92, 0, }, /* 660 */ + { 92, 21, 12, 0, 0, 92, 0, }, /* 661 */ + { 87, 7, 12, 0, 0, 87, 0, }, /* 662 */ + { 87, 10, 5, 0, 0, 87, 0, }, /* 663 */ + { 87, 12, 3, 0, 0, 87, 0, }, /* 664 */ + { 87, 21, 12, 0, 0, 87, 0, }, /* 665 */ + { 87, 6, 12, 0, 0, 87, 0, }, /* 666 */ + { 34, 5, 12, 0, -928, 34, 0, }, /* 667 */ + { 9, 5, 12, 0, -38864, 9, 0, }, /* 668 */ + { 87, 13, 12, 0, 0, 87, 0, }, /* 669 */ + { 24, 7, 9, 0, 0, 24, 0, }, /* 670 */ + { 24, 7, 10, 0, 0, 24, 0, }, /* 671 */ + { 0, 4, 12, 0, 0, 0, 0, }, /* 672 */ + { 0, 3, 12, 0, 0, 0, 0, }, /* 673 */ + { 26, 25, 12, 0, 0, 26, 0, }, /* 674 */ + { 1, 24, 12, 0, 0, 1, 0, }, /* 675 */ + { 1, 7, 12, 0, 0, -10, 0, }, /* 676 */ + { 1, 26, 12, 0, 0, -10, 0, }, /* 677 */ + { 10, 6, 3, 0, 0, -64, 0, }, /* 678 */ + { 36, 7, 12, 0, 0, 36, 0, }, /* 679 */ + { 10, 21, 12, 0, 0, -22, 0, }, /* 680 */ + { 10, 15, 12, 0, 0, -92, 0, }, /* 681 */ + { 10, 26, 12, 0, 0, -22, 0, }, /* 682 */ + { 20, 14, 12, 0, 0, 20, 0, }, /* 683 */ + { 20, 15, 12, 0, 0, 20, 0, }, /* 684 */ + { 20, 26, 12, 0, 0, 20, 0, }, /* 685 */ + { 71, 7, 12, 0, 0, 71, 0, }, /* 686 */ + { 67, 7, 12, 0, 0, 67, 0, }, /* 687 */ + { 28, 12, 3, 0, 0, -1, 0, }, /* 688 */ + { 10, 15, 12, 0, 0, -1, 0, }, /* 689 */ + { 42, 7, 12, 0, 0, 42, 0, }, /* 690 */ + { 42, 15, 12, 0, 0, 42, 0, }, /* 691 */ + { 19, 7, 12, 0, 0, 19, 0, }, /* 692 */ + { 19, 14, 12, 0, 0, 19, 0, }, /* 693 */ + { 118, 7, 12, 0, 0, 118, 0, }, /* 694 */ + { 118, 12, 3, 0, 0, 118, 0, }, /* 695 */ + { 60, 7, 12, 0, 0, 60, 0, }, /* 696 */ + { 60, 21, 12, 0, 0, 60, 0, }, /* 697 */ + { 43, 7, 12, 0, 0, 43, 0, }, /* 698 */ + { 43, 21, 12, 0, 0, 43, 0, }, /* 699 */ + { 43, 14, 12, 0, 0, 43, 0, }, /* 700 */ + { 14, 9, 12, 0, 40, 14, 0, }, /* 701 */ + { 14, 5, 12, 0, -40, 14, 0, }, /* 702 */ + { 47, 7, 12, 0, 0, 47, 0, }, /* 703 */ + { 45, 7, 12, 0, 0, 45, 0, }, /* 704 */ + { 45, 13, 12, 0, 0, 45, 0, }, /* 705 */ + { 136, 9, 12, 0, 40, 136, 0, }, /* 706 */ + { 136, 5, 12, 0, -40, 136, 0, }, /* 707 */ + { 106, 7, 12, 0, 0, 106, 0, }, /* 708 */ + { 104, 7, 12, 0, 0, 104, 0, }, /* 709 */ + { 104, 21, 12, 0, 0, 104, 0, }, /* 710 */ + { 110, 7, 12, 0, 0, 110, 0, }, /* 711 */ + { 12, 7, 12, 0, 0, 12, 0, }, /* 712 */ + { 81, 7, 12, 0, 0, 81, 0, }, /* 713 */ + { 81, 21, 12, 0, 0, 81, 0, }, /* 714 */ + { 81, 15, 12, 0, 0, 81, 0, }, /* 715 */ + { 120, 7, 12, 0, 0, 120, 0, }, /* 716 */ + { 120, 26, 12, 0, 0, 120, 0, }, /* 717 */ + { 120, 15, 12, 0, 0, 120, 0, }, /* 718 */ + { 116, 7, 12, 0, 0, 116, 0, }, /* 719 */ + { 116, 15, 12, 0, 0, 116, 0, }, /* 720 */ + { 128, 7, 12, 0, 0, 128, 0, }, /* 721 */ + { 128, 15, 12, 0, 0, 128, 0, }, /* 722 */ + { 66, 7, 12, 0, 0, 66, 0, }, /* 723 */ + { 66, 15, 12, 0, 0, 66, 0, }, /* 724 */ + { 66, 21, 12, 0, 0, 66, 0, }, /* 725 */ + { 72, 7, 12, 0, 0, 72, 0, }, /* 726 */ + { 72, 21, 12, 0, 0, 72, 0, }, /* 727 */ + { 98, 7, 12, 0, 0, 98, 0, }, /* 728 */ + { 97, 7, 12, 0, 0, 97, 0, }, /* 729 */ + { 97, 15, 12, 0, 0, 97, 0, }, /* 730 */ + { 31, 7, 12, 0, 0, 31, 0, }, /* 731 */ + { 31, 12, 3, 0, 0, 31, 0, }, /* 732 */ + { 31, 15, 12, 0, 0, 31, 0, }, /* 733 */ + { 31, 21, 12, 0, 0, 31, 0, }, /* 734 */ + { 88, 7, 12, 0, 0, 88, 0, }, /* 735 */ + { 88, 15, 12, 0, 0, 88, 0, }, /* 736 */ + { 88, 21, 12, 0, 0, 88, 0, }, /* 737 */ + { 117, 7, 12, 0, 0, 117, 0, }, /* 738 */ + { 117, 15, 12, 0, 0, 117, 0, }, /* 739 */ + { 112, 7, 12, 0, 0, 112, 0, }, /* 740 */ + { 112, 26, 12, 0, 0, 112, 0, }, /* 741 */ + { 112, 12, 3, 0, 0, 112, 0, }, /* 742 */ + { 112, 15, 12, 0, 0, 112, 0, }, /* 743 */ + { 112, 21, 12, 0, 0, 112, 0, }, /* 744 */ + { 78, 7, 12, 0, 0, 78, 0, }, /* 745 */ + { 78, 21, 12, 0, 0, 78, 0, }, /* 746 */ + { 83, 7, 12, 0, 0, 83, 0, }, /* 747 */ + { 83, 15, 12, 0, 0, 83, 0, }, /* 748 */ + { 82, 7, 12, 0, 0, 82, 0, }, /* 749 */ + { 82, 15, 12, 0, 0, 82, 0, }, /* 750 */ + { 121, 7, 12, 0, 0, 121, 0, }, /* 751 */ + { 121, 21, 12, 0, 0, 121, 0, }, /* 752 */ + { 121, 15, 12, 0, 0, 121, 0, }, /* 753 */ + { 89, 7, 12, 0, 0, 89, 0, }, /* 754 */ + { 130, 9, 12, 0, 64, 130, 0, }, /* 755 */ + { 130, 5, 12, 0, -64, 130, 0, }, /* 756 */ + { 130, 15, 12, 0, 0, 130, 0, }, /* 757 */ + { 144, 7, 12, 0, 0, 144, 0, }, /* 758 */ + { 144, 12, 3, 0, 0, 144, 0, }, /* 759 */ + { 144, 13, 12, 0, 0, 144, 0, }, /* 760 */ + { 1, 15, 12, 0, 0, 1, 0, }, /* 761 */ + { 156, 7, 12, 0, 0, 156, 0, }, /* 762 */ + { 156, 12, 3, 0, 0, 156, 0, }, /* 763 */ + { 156, 17, 12, 0, 0, 156, 0, }, /* 764 */ + { 147, 7, 12, 0, 0, 147, 0, }, /* 765 */ + { 147, 15, 12, 0, 0, 147, 0, }, /* 766 */ + { 148, 7, 12, 0, 0, 148, 0, }, /* 767 */ + { 148, 12, 3, 0, 0, 148, 0, }, /* 768 */ + { 148, 15, 12, 0, 0, 148, 0, }, /* 769 */ + { 148, 21, 12, 0, 0, 148, 0, }, /* 770 */ + { 153, 7, 12, 0, 0, 153, 0, }, /* 771 */ + { 153, 15, 12, 0, 0, 153, 0, }, /* 772 */ + { 149, 7, 12, 0, 0, 149, 0, }, /* 773 */ + { 94, 10, 5, 0, 0, 94, 0, }, /* 774 */ + { 94, 12, 3, 0, 0, 94, 0, }, /* 775 */ + { 94, 7, 12, 0, 0, 94, 0, }, /* 776 */ + { 94, 21, 12, 0, 0, 94, 0, }, /* 777 */ + { 94, 15, 12, 0, 0, 94, 0, }, /* 778 */ + { 94, 13, 12, 0, 0, 94, 0, }, /* 779 */ + { 85, 12, 3, 0, 0, 85, 0, }, /* 780 */ + { 85, 10, 5, 0, 0, 85, 0, }, /* 781 */ + { 85, 7, 12, 0, 0, 85, 0, }, /* 782 */ + { 85, 21, 12, 0, 0, 85, 0, }, /* 783 */ + { 85, 1, 4, 0, 0, 85, 0, }, /* 784 */ + { 101, 7, 12, 0, 0, 101, 0, }, /* 785 */ + { 101, 13, 12, 0, 0, 101, 0, }, /* 786 */ + { 96, 12, 3, 0, 0, 96, 0, }, /* 787 */ + { 96, 7, 12, 0, 0, 96, 0, }, /* 788 */ + { 96, 10, 5, 0, 0, 96, 0, }, /* 789 */ + { 96, 13, 12, 0, 0, 96, 0, }, /* 790 */ + { 96, 21, 12, 0, 0, 96, 0, }, /* 791 */ + { 111, 7, 12, 0, 0, 111, 0, }, /* 792 */ + { 111, 12, 3, 0, 0, 111, 0, }, /* 793 */ + { 111, 21, 12, 0, 0, 111, 0, }, /* 794 */ + { 100, 12, 3, 0, 0, 100, 0, }, /* 795 */ + { 100, 10, 5, 0, 0, 100, 0, }, /* 796 */ + { 100, 7, 12, 0, 0, 100, 0, }, /* 797 */ + { 100, 7, 4, 0, 0, 100, 0, }, /* 798 */ + { 100, 21, 12, 0, 0, 100, 0, }, /* 799 */ + { 100, 13, 12, 0, 0, 100, 0, }, /* 800 */ + { 48, 15, 12, 0, 0, 48, 0, }, /* 801 */ + { 108, 7, 12, 0, 0, 108, 0, }, /* 802 */ + { 108, 10, 5, 0, 0, 108, 0, }, /* 803 */ + { 108, 12, 3, 0, 0, 108, 0, }, /* 804 */ + { 108, 21, 12, 0, 0, 108, 0, }, /* 805 */ + { 129, 7, 12, 0, 0, 129, 0, }, /* 806 */ + { 129, 21, 12, 0, 0, 129, 0, }, /* 807 */ + { 109, 7, 12, 0, 0, 109, 0, }, /* 808 */ + { 109, 12, 3, 0, 0, 109, 0, }, /* 809 */ + { 109, 10, 5, 0, 0, 109, 0, }, /* 810 */ + { 109, 13, 12, 0, 0, 109, 0, }, /* 811 */ + { 107, 12, 3, 0, 0, 107, 0, }, /* 812 */ + { 107, 12, 3, 0, 0, -52, 0, }, /* 813 */ + { 107, 10, 5, 0, 0, 107, 0, }, /* 814 */ + { 107, 10, 5, 0, 0, -52, 0, }, /* 815 */ + { 107, 7, 12, 0, 0, 107, 0, }, /* 816 */ + { 28, 12, 3, 0, 0, -52, 0, }, /* 817 */ + { 107, 10, 3, 0, 0, 107, 0, }, /* 818 */ + { 135, 7, 12, 0, 0, 135, 0, }, /* 819 */ + { 135, 10, 5, 0, 0, 135, 0, }, /* 820 */ + { 135, 12, 3, 0, 0, 135, 0, }, /* 821 */ + { 135, 21, 12, 0, 0, 135, 0, }, /* 822 */ + { 135, 13, 12, 0, 0, 135, 0, }, /* 823 */ + { 124, 7, 12, 0, 0, 124, 0, }, /* 824 */ + { 124, 10, 3, 0, 0, 124, 0, }, /* 825 */ + { 124, 10, 5, 0, 0, 124, 0, }, /* 826 */ + { 124, 12, 3, 0, 0, 124, 0, }, /* 827 */ + { 124, 21, 12, 0, 0, 124, 0, }, /* 828 */ + { 124, 13, 12, 0, 0, 124, 0, }, /* 829 */ + { 123, 7, 12, 0, 0, 123, 0, }, /* 830 */ + { 123, 10, 3, 0, 0, 123, 0, }, /* 831 */ + { 123, 10, 5, 0, 0, 123, 0, }, /* 832 */ + { 123, 12, 3, 0, 0, 123, 0, }, /* 833 */ + { 123, 21, 12, 0, 0, 123, 0, }, /* 834 */ + { 114, 7, 12, 0, 0, 114, 0, }, /* 835 */ + { 114, 10, 5, 0, 0, 114, 0, }, /* 836 */ + { 114, 12, 3, 0, 0, 114, 0, }, /* 837 */ + { 114, 21, 12, 0, 0, 114, 0, }, /* 838 */ + { 114, 13, 12, 0, 0, 114, 0, }, /* 839 */ + { 102, 7, 12, 0, 0, 102, 0, }, /* 840 */ + { 102, 12, 3, 0, 0, 102, 0, }, /* 841 */ + { 102, 10, 5, 0, 0, 102, 0, }, /* 842 */ + { 102, 13, 12, 0, 0, 102, 0, }, /* 843 */ + { 126, 7, 12, 0, 0, 126, 0, }, /* 844 */ + { 126, 12, 3, 0, 0, 126, 0, }, /* 845 */ + { 126, 10, 5, 0, 0, 126, 0, }, /* 846 */ + { 126, 13, 12, 0, 0, 126, 0, }, /* 847 */ + { 126, 15, 12, 0, 0, 126, 0, }, /* 848 */ + { 126, 21, 12, 0, 0, 126, 0, }, /* 849 */ + { 126, 26, 12, 0, 0, 126, 0, }, /* 850 */ + { 142, 7, 12, 0, 0, 142, 0, }, /* 851 */ + { 142, 10, 5, 0, 0, 142, 0, }, /* 852 */ + { 142, 12, 3, 0, 0, 142, 0, }, /* 853 */ + { 142, 21, 12, 0, 0, 142, 0, }, /* 854 */ + { 125, 9, 12, 0, 32, 125, 0, }, /* 855 */ + { 125, 5, 12, 0, -32, 125, 0, }, /* 856 */ + { 125, 13, 12, 0, 0, 125, 0, }, /* 857 */ + { 125, 15, 12, 0, 0, 125, 0, }, /* 858 */ + { 125, 7, 12, 0, 0, 125, 0, }, /* 859 */ + { 154, 7, 12, 0, 0, 154, 0, }, /* 860 */ + { 154, 10, 3, 0, 0, 154, 0, }, /* 861 */ + { 154, 10, 5, 0, 0, 154, 0, }, /* 862 */ + { 154, 12, 3, 0, 0, 154, 0, }, /* 863 */ + { 154, 7, 4, 0, 0, 154, 0, }, /* 864 */ + { 154, 21, 12, 0, 0, 154, 0, }, /* 865 */ + { 154, 13, 12, 0, 0, 154, 0, }, /* 866 */ + { 150, 7, 12, 0, 0, 150, 0, }, /* 867 */ + { 150, 10, 5, 0, 0, 150, 0, }, /* 868 */ + { 150, 12, 3, 0, 0, 150, 0, }, /* 869 */ + { 150, 21, 12, 0, 0, 150, 0, }, /* 870 */ + { 141, 7, 12, 0, 0, 141, 0, }, /* 871 */ + { 141, 12, 3, 0, 0, 141, 0, }, /* 872 */ + { 141, 10, 5, 0, 0, 141, 0, }, /* 873 */ + { 141, 7, 4, 0, 0, 141, 0, }, /* 874 */ + { 141, 21, 12, 0, 0, 141, 0, }, /* 875 */ + { 140, 7, 12, 0, 0, 140, 0, }, /* 876 */ + { 140, 12, 3, 0, 0, 140, 0, }, /* 877 */ + { 140, 10, 5, 0, 0, 140, 0, }, /* 878 */ + { 140, 7, 4, 0, 0, 140, 0, }, /* 879 */ + { 140, 21, 12, 0, 0, 140, 0, }, /* 880 */ + { 122, 7, 12, 0, 0, 122, 0, }, /* 881 */ + { 133, 7, 12, 0, 0, 133, 0, }, /* 882 */ + { 133, 10, 5, 0, 0, 133, 0, }, /* 883 */ + { 133, 12, 3, 0, 0, 133, 0, }, /* 884 */ + { 133, 21, 12, 0, 0, 133, 0, }, /* 885 */ + { 133, 13, 12, 0, 0, 133, 0, }, /* 886 */ + { 133, 15, 12, 0, 0, 133, 0, }, /* 887 */ + { 134, 21, 12, 0, 0, 134, 0, }, /* 888 */ + { 134, 7, 12, 0, 0, 134, 0, }, /* 889 */ + { 134, 12, 3, 0, 0, 134, 0, }, /* 890 */ + { 134, 10, 5, 0, 0, 134, 0, }, /* 891 */ + { 138, 7, 12, 0, 0, 138, 0, }, /* 892 */ + { 138, 12, 3, 0, 0, 138, 0, }, /* 893 */ + { 138, 7, 4, 0, 0, 138, 0, }, /* 894 */ + { 138, 13, 12, 0, 0, 138, 0, }, /* 895 */ + { 143, 7, 12, 0, 0, 143, 0, }, /* 896 */ + { 143, 10, 5, 0, 0, 143, 0, }, /* 897 */ + { 143, 12, 3, 0, 0, 143, 0, }, /* 898 */ + { 143, 13, 12, 0, 0, 143, 0, }, /* 899 */ + { 145, 7, 12, 0, 0, 145, 0, }, /* 900 */ + { 145, 12, 3, 0, 0, 145, 0, }, /* 901 */ + { 145, 10, 5, 0, 0, 145, 0, }, /* 902 */ + { 145, 21, 12, 0, 0, 145, 0, }, /* 903 */ + { 54, 15, 12, 0, 0, 54, 0, }, /* 904 */ + { 54, 21, 12, 0, 0, 54, 0, }, /* 905 */ + { 63, 7, 12, 0, 0, 63, 0, }, /* 906 */ + { 63, 14, 12, 0, 0, 63, 0, }, /* 907 */ + { 63, 21, 12, 0, 0, 63, 0, }, /* 908 */ + { 80, 7, 12, 0, 0, 80, 0, }, /* 909 */ + { 80, 1, 2, 0, 0, 80, 0, }, /* 910 */ + { 127, 7, 12, 0, 0, 127, 0, }, /* 911 */ + { 115, 7, 12, 0, 0, 115, 0, }, /* 912 */ + { 115, 13, 12, 0, 0, 115, 0, }, /* 913 */ + { 115, 21, 12, 0, 0, 115, 0, }, /* 914 */ + { 103, 7, 12, 0, 0, 103, 0, }, /* 915 */ + { 103, 12, 3, 0, 0, 103, 0, }, /* 916 */ + { 103, 21, 12, 0, 0, 103, 0, }, /* 917 */ + { 119, 7, 12, 0, 0, 119, 0, }, /* 918 */ + { 119, 12, 3, 0, 0, 119, 0, }, /* 919 */ + { 119, 21, 12, 0, 0, 119, 0, }, /* 920 */ + { 119, 26, 12, 0, 0, 119, 0, }, /* 921 */ + { 119, 6, 12, 0, 0, 119, 0, }, /* 922 */ + { 119, 13, 12, 0, 0, 119, 0, }, /* 923 */ + { 119, 15, 12, 0, 0, 119, 0, }, /* 924 */ + { 146, 9, 12, 0, 32, 146, 0, }, /* 925 */ + { 146, 5, 12, 0, -32, 146, 0, }, /* 926 */ + { 146, 15, 12, 0, 0, 146, 0, }, /* 927 */ + { 146, 21, 12, 0, 0, 146, 0, }, /* 928 */ + { 99, 7, 12, 0, 0, 99, 0, }, /* 929 */ + { 99, 12, 3, 0, 0, 99, 0, }, /* 930 */ + { 99, 10, 5, 0, 0, 99, 0, }, /* 931 */ + { 99, 6, 12, 0, 0, 99, 0, }, /* 932 */ + { 137, 6, 12, 0, 0, 137, 0, }, /* 933 */ + { 139, 6, 12, 0, 0, 139, 0, }, /* 934 */ + { 155, 12, 3, 0, 0, 155, 0, }, /* 935 */ + { 23, 10, 5, 0, 0, 23, 0, }, /* 936 */ + { 137, 7, 12, 0, 0, 137, 0, }, /* 937 */ + { 155, 7, 12, 0, 0, 155, 0, }, /* 938 */ + { 139, 7, 12, 0, 0, 139, 0, }, /* 939 */ + { 105, 7, 12, 0, 0, 105, 0, }, /* 940 */ + { 105, 26, 12, 0, 0, 105, 0, }, /* 941 */ + { 105, 12, 3, 0, 0, 105, 0, }, /* 942 */ + { 105, 21, 12, 0, 0, 105, 0, }, /* 943 */ + { 10, 1, 2, 0, 0, 105, 0, }, /* 944 */ + { 10, 10, 3, 0, 0, 10, 0, }, /* 945 */ + { 10, 10, 5, 0, 0, 10, 0, }, /* 946 */ + { 20, 12, 3, 0, 0, 20, 0, }, /* 947 */ + { 131, 26, 12, 0, 0, 131, 0, }, /* 948 */ + { 131, 12, 3, 0, 0, 131, 0, }, /* 949 */ + { 131, 21, 12, 0, 0, 131, 0, }, /* 950 */ + { 18, 12, 3, 0, 0, 18, 0, }, /* 951 */ + { 151, 7, 12, 0, 0, 151, 0, }, /* 952 */ + { 151, 12, 3, 0, 0, 151, 0, }, /* 953 */ + { 151, 6, 12, 0, 0, 151, 0, }, /* 954 */ + { 151, 13, 12, 0, 0, 151, 0, }, /* 955 */ + { 151, 26, 12, 0, 0, 151, 0, }, /* 956 */ + { 152, 7, 12, 0, 0, 152, 0, }, /* 957 */ + { 152, 12, 3, 0, 0, 152, 0, }, /* 958 */ + { 152, 13, 12, 0, 0, 152, 0, }, /* 959 */ + { 152, 23, 12, 0, 0, 152, 0, }, /* 960 */ + { 113, 7, 12, 0, 0, 113, 0, }, /* 961 */ + { 113, 15, 12, 0, 0, 113, 0, }, /* 962 */ + { 113, 12, 3, 0, 0, 113, 0, }, /* 963 */ + { 132, 9, 12, 0, 34, 132, 0, }, /* 964 */ + { 132, 5, 12, 0, -34, 132, 0, }, /* 965 */ + { 132, 12, 3, 0, 0, 132, 0, }, /* 966 */ + { 132, 6, 12, 0, 0, 132, 0, }, /* 967 */ + { 132, 13, 12, 0, 0, 132, 0, }, /* 968 */ + { 132, 21, 12, 0, 0, 132, 0, }, /* 969 */ + { 0, 2, 14, 0, 0, 0, 0, }, /* 970 */ + { 10, 26, 11, 0, 0, 10, 0, }, /* 971 */ + { 27, 26, 12, 0, 0, 27, 0, }, /* 972 */ + { 10, 24, 3, 0, 0, 10, 0, }, /* 973 */ + { 10, 1, 3, 0, 0, 10, 0, }, /* 974 */ }; const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ @@ -1150,37 +1204,37 @@ const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+F000 */ 126,126, 98, 98,127,128,129,130,131,131,132,133,134,135,136,137, /* U+F800 */ 138,139,140,141,142,143,144,145,146,147,148,142,149,149,150,142, /* U+10000 */ -151,152,153,154,155,156,157,158,159,160,161,142,162,142,163,142, /* U+10800 */ -164,165,166,167,168,169,170,142,171,172,142,173,174,175,176,142, /* U+11000 */ -177,178,142,142,179,180,142,142,181,182,183,184,142,185,142,142, /* U+11800 */ -186,186,186,186,186,186,186,187,188,186,189,142,142,142,142,142, /* U+12000 */ +151,152,153,154,155,156,157,158,159,160,161,142,162,163,164,165, /* U+10800 */ +166,167,168,169,170,171,172,142,173,174,142,175,176,177,178,142, /* U+11000 */ +179,180,181,182,183,184,142,142,185,186,187,188,142,189,142,190, /* U+11800 */ +191,191,191,191,191,191,191,192,193,191,194,142,142,142,142,142, /* U+12000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+12800 */ -190,190,190,190,190,190,190,190,191,142,142,142,142,142,142,142, /* U+13000 */ +195,195,195,195,195,195,195,195,196,142,142,142,142,142,142,142, /* U+13000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+13800 */ -142,142,142,142,142,142,142,142,192,192,192,192,193,142,142,142, /* U+14000 */ +142,142,142,142,142,142,142,142,197,197,197,197,198,142,142,142, /* U+14000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+14800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+15000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+15800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+16000 */ -194,194,194,194,195,196,197,198,142,142,142,142,199,200,201,202, /* U+16800 */ -203,203,203,203,203,203,203,203,203,203,203,203,203,203,203,203, /* U+17000 */ -203,203,203,203,203,203,203,203,203,203,203,203,203,203,203,203, /* U+17800 */ -203,203,203,203,203,203,203,203,203,203,203,203,203,203,203,204, /* U+18000 */ -203,203,203,203,203,205,142,142,142,142,142,142,142,142,142,142, /* U+18800 */ +199,199,199,199,200,201,202,203,142,142,142,142,204,205,206,207, /* U+16800 */ +208,208,208,208,208,208,208,208,208,208,208,208,208,208,208,208, /* U+17000 */ +208,208,208,208,208,208,208,208,208,208,208,208,208,208,208,208, /* U+17800 */ +208,208,208,208,208,208,208,208,208,208,208,208,208,208,208,209, /* U+18000 */ +208,208,208,208,208,208,210,210,210,211,212,142,142,142,142,142, /* U+18800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+19000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+19800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+1A000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+1A800 */ -206,207,208,209,209,210,142,142,142,142,142,142,142,142,142,142, /* U+1B000 */ -142,142,142,142,142,142,142,142,211,212,142,142,142,142,142,142, /* U+1B800 */ +213,214,215,216,216,217,142,142,142,142,142,142,142,142,142,142, /* U+1B000 */ +142,142,142,142,142,142,142,142,218,219,142,142,142,142,142,142, /* U+1B800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+1C000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+1C800 */ - 71,213,214,215,216,217,218,142,219,220,221,222,223,224,225,226, /* U+1D000 */ -227,227,227,227,228,229,142,142,142,142,142,142,142,142,142,142, /* U+1D800 */ -230,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+1E000 */ -231,232,233,142,142,142,142,142,234,235,142,142,236,237,142,142, /* U+1E800 */ -238,239,240,241,242,243,244,245,244,244,246,244,247,248,249,250, /* U+1F000 */ -251,252,253,254,255,243,243,243,243,243,243,243,243,243,243,256, /* U+1F800 */ + 71,220,221,222,223,224,225,142,226,227,228,229,230,231,232,233, /* U+1D000 */ +234,234,234,234,235,236,142,142,142,142,142,142,142,142,142,142, /* U+1D800 */ +237,142,238,142,142,239,142,142,142,142,142,142,142,142,142,142, /* U+1E000 */ +240,241,242,142,142,142,142,142,243,244,245,142,246,247,142,142, /* U+1E800 */ +248,249,250,251,252,253,254,255,254,254,256,254,257,258,259,260, /* U+1F000 */ +261,262,263,264,265,266, 71,267,253,253,253,253,253,253,253,268, /* U+1F800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+20000 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+20800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+21000 */ @@ -1201,21 +1255,21 @@ const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+28800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+29000 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+29800 */ - 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,257, 98, 98, /* U+2A000 */ + 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,269, 98, 98, /* U+2A000 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2A800 */ - 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,258, 98, /* U+2B000 */ -259, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2B800 */ + 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,270, 98, /* U+2B000 */ +271, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2B800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2C000 */ - 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,260, 98, 98, /* U+2C800 */ + 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98,272, 98, 98, /* U+2C800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2D000 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2D800 */ 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+2E000 */ - 98, 98, 98, 98, 98, 98, 98,261,142,142,142,142,142,142,142,142, /* U+2E800 */ + 98, 98, 98, 98, 98, 98, 98,273,142,142,142,142,142,142,142,142, /* U+2E800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+2F000 */ - 98, 98, 98, 98,262,142,142,142,142,142,142,142,142,142,142,142, /* U+2F800 */ -142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+30000 */ -142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+30800 */ -142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+31000 */ + 98, 98, 98, 98,274,142,142,142,142,142,142,142,142,142,142,142, /* U+2F800 */ + 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+30000 */ + 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, 98, /* U+30800 */ + 98, 98, 98, 98, 98, 98,275,142,142,142,142,142,142,142,142,142, /* U+31000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+31800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+32000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+32800 */ @@ -1565,8 +1619,8 @@ const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+DE800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+DF000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+DF800 */ -263,264,265,266,264,264,264,264,264,264,264,264,264,264,264,264, /* U+E0000 */ -264,264,264,264,264,264,264,264,264,264,264,264,264,264,264,264, /* U+E0800 */ +276,277,278,279,277,277,277,277,277,277,277,277,277,277,277,277, /* U+E0000 */ +277,277,277,277,277,277,277,277,277,277,277,277,277,277,277,277, /* U+E0800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+E1000 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+E1800 */ 142,142,142,142,142,142,142,142,142,142,142,142,142,142,142,142, /* U+E2000 */ @@ -1628,7 +1682,7 @@ const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+FE000 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+FE800 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+FF000 */ -126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,267, /* U+FF800 */ +126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,280, /* U+FF800 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+100000 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+100800 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+101000 */ @@ -1660,10 +1714,10 @@ const uint16_t PRIV(ucd_stage1)[] = { /* 17408 bytes */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+10E000 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+10E800 */ 126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,126, /* U+10F000 */ -126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,267, /* U+10F800 */ +126,126,126,126,126,126,126,126,126,126,126,126,126,126,126,280, /* U+10F800 */ }; -const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ +const uint16_t PRIV(ucd_stage2)[] = { /* 71936 bytes, block = 128 */ /* block 0 */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1715,78 +1769,78 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 35, 97, 98, 35, 35, 99, 35, 35, 35, 35, 35, 35, 35,100, 35, 35, /* block 5 */ -101, 35, 35,101, 35, 35, 35,102,101,103,104,104,105, 35, 35, 35, - 35, 35,106, 35, 22, 35, 35, 35, 35, 35, 35, 35, 35,107,108, 35, +101, 35,102,101, 35, 35, 35,103,101,104,105,105,106, 35, 35, 35, + 35, 35,107, 35, 22, 35, 35, 35, 35, 35, 35, 35, 35,108,109, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, -109,109,109,109,109,109,109,109,109,110,110,110,110,110,110,110, -110,110, 15, 15, 15, 15,110,110,110,110,110,110,110,110,110,110, -110,110, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, -109,109,109,109,109, 15, 15, 15, 15, 15,111,111,110, 15,110, 15, +110,110,110,110,110,110,110,110,110,111,111,111,111,111,111,111, +111,111, 15, 15, 15, 15,111,111,111,111,111,111,111,111,111,111, +111,111, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, +110,110,110,110,110, 15, 15, 15, 15, 15,112,112,111, 15,111, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, /* block 6 */ -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,113,112,112,114,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,115,115,115,115,115,115,115,115,115,115,115,115,115, -116,117,116,117,110,118,116,117,119,119,120,121,121,121, 5,122, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,114,113,113,115,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,116,116,116,116,116,116,116,116,116,116,116,116,116, +117,118,117,118,111,119,117,118,120,120,121,122,122,122, 5,123, /* block 7 */ -119,119,119,119,118, 15,123, 5,124,124,124,119,125,119,126,126, -127,128,129,128,128,130,128,128,131,132,133,128,134,128,128,128, -135,136,119,137,128,128,138,128,128,139,128,128,140,141,141,141, -127,142,143,142,142,144,142,142,145,146,147,142,148,142,142,142, -149,150,151,152,142,142,153,142,142,154,142,142,155,156,156,157, -158,159,160,160,160,161,162,163,116,117,116,117,116,117,116,117, -116,117,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -166,167,168,169,170,171,172,116,117,173,116,117,127,174,174,174, +120,120,120,120,119, 15,124, 5,125,125,125,120,126,120,127,127, +128,129,130,129,129,131,129,129,132,133,134,129,135,129,129,129, +136,137,120,138,129,129,139,129,129,140,129,129,141,142,142,142, +128,143,144,143,143,145,143,143,146,147,148,143,149,143,143,143, +150,151,152,153,143,143,154,143,143,155,143,143,156,157,157,158, +159,160,161,161,161,162,163,164,117,118,117,118,117,118,117,118, +117,118,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +167,168,169,170,171,172,173,117,118,174,117,118,128,175,175,175, /* block 8 */ -175,175,175,175,175,175,175,175,175,175,175,175,175,175,175,175, -176,176,177,176,178,176,176,176,176,176,176,176,176,176,179,176, -176,180,181,176,176,176,176,176,176,176,182,176,176,176,176,176, -183,183,184,183,185,183,183,183,183,183,183,183,183,183,186,183, -183,187,188,183,183,183,183,183,183,183,189,183,183,183,183,183, -190,190,190,190,190,190,190,190,190,190,190,190,190,190,190,190, -191,192,193,194,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, +176,176,176,176,176,176,176,176,176,176,176,176,176,176,176,176, +177,177,178,177,179,177,177,177,177,177,177,177,177,177,180,177, +177,181,182,177,177,177,177,177,177,177,183,177,177,177,177,177, +184,184,185,184,186,184,184,184,184,184,184,184,184,184,187,184, +184,188,189,184,184,184,184,184,184,184,190,184,184,184,184,184, +191,191,191,191,191,191,191,191,191,191,191,191,191,191,191,191, +192,193,194,195,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, /* block 9 */ -191,192,195,196,197,198,198,197,199,199,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -200,191,192,191,192,191,192,191,192,191,192,191,192,191,192,201, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, +192,193,196,197,198,199,199,198,200,200,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +201,192,193,192,193,192,193,192,193,192,193,192,193,192,193,202, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, /* block 10 */ -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -119,202,202,202,202,202,202,202,202,202,202,202,202,202,202,202, -202,202,202,202,202,202,202,202,202,202,202,202,202,202,202,202, -202,202,202,202,202,202,202,119,119,203,204,204,204,204,204,204, -205,206,206,206,206,206,206,206,206,206,206,206,206,206,206,206, -206,206,206,206,206,206,206,206,206,206,206,206,206,206,206,206, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +120,203,203,203,203,203,203,203,203,203,203,203,203,203,203,203, +203,203,203,203,203,203,203,203,203,203,203,203,203,203,203,203, +203,203,203,203,203,203,203,120,120,204,205,205,205,205,205,205, +206,207,207,207,207,207,207,207,207,207,207,207,207,207,207,207, +207,207,207,207,207,207,207,207,207,207,207,207,207,207,207,207, /* block 11 */ -206,206,206,206,206,206,206,205,205,207,208,119,119,209,209,210, -119,211,211,211,211,211,211,211,211,211,211,211,211,211,211,211, +207,207,207,207,207,207,207,206,206,205,208,120,120,209,209,210, +120,211,211,211,211,211,211,211,211,211,211,211,211,211,211,211, 211,211,211,211,211,211,211,211,211,211,211,211,211,211,211,211, 211,211,211,211,211,211,211,211,211,211,211,211,211,211,212,211, -213,211,211,213,211,211,213,211,119,119,119,119,119,119,119,119, +213,211,211,213,211,211,213,211,120,120,120,120,120,120,120,120, 214,214,214,214,214,214,214,214,214,214,214,214,214,214,214,214, -214,214,214,214,214,214,214,214,214,214,214,119,119,119,119,214, -214,214,214,213,213,119,119,119,119,119,119,119,119,119,119,119, +214,214,214,214,214,214,214,214,214,214,214,120,120,120,120,214, +214,214,214,213,213,120,120,120,120,120,120,120,120,120,120,120, /* block 12 */ 215,215,215,215,215,216,217,217,217,218,218,219,220,218,221,221, -222,222,222,222,222,222,222,222,222,222,222,220,223,119,218,220, +222,222,222,222,222,222,222,222,222,222,222,220,223,120,218,220, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 225,224,224,224,224,224,224,224,224,224,224,226,226,226,226,226, @@ -1805,11 +1859,11 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 230,230,230,230,230,230,230,230,230,230,224,224,224,221,221,224, /* block 14 */ -231,231,231,231,231,231,231,231,231,231,231,231,231,231,119,232, +231,231,231,231,231,231,231,231,231,231,231,231,231,231,120,232, 233,234,233,233,233,233,233,233,233,233,233,233,233,233,233,233, 233,233,233,233,233,233,233,233,233,233,233,233,233,233,233,233, 234,234,234,234,234,234,234,234,234,234,234,234,234,234,234,234, -234,234,234,234,234,234,234,234,234,234,234,119,119,233,233,233, +234,234,234,234,234,234,234,234,234,234,234,120,120,233,233,233, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, @@ -1818,29 +1872,29 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 235,235,235,235,235,235,235,235,235,235,235,235,235,235,235,235, 235,235,235,235,235,235,235,235,235,235,235,235,235,235,235,235, 235,235,235,235,235,235,236,236,236,236,236,236,236,236,236,236, -236,235,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +236,235,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 237,237,237,237,237,237,237,237,237,237,238,238,238,238,238,238, 238,238,238,238,238,238,238,238,238,238,238,238,238,238,238,238, 238,238,238,238,238,238,238,238,238,238,238,239,239,239,239,239, -239,239,239,239,240,240,241,242,242,242,240,119,119,239,243,243, +239,239,239,239,240,240,241,242,242,242,240,120,120,239,243,243, /* block 16 */ 244,244,244,244,244,244,244,244,244,244,244,244,244,244,244,244, 244,244,244,244,244,244,245,245,245,245,246,245,245,245,245,245, -245,245,245,245,246,245,245,245,246,245,245,245,245,245,119,119, -247,247,247,247,247,247,247,247,247,247,247,247,247,247,247,119, +245,245,245,245,246,245,245,245,246,245,245,245,245,245,120,120, +247,247,247,247,247,247,247,247,247,247,247,247,247,247,247,120, 248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248, -248,248,248,248,248,248,248,248,248,249,249,249,119,119,250,119, -233,233,233,233,233,233,233,233,233,233,233,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +248,248,248,248,248,248,248,248,248,249,249,249,120,120,250,120, +233,233,233,233,233,233,233,233,233,233,233,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 17 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -224,224,224,224,224,119,224,224,224,224,224,224,224,224,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,222,222,222,222,222,222,222,222,222,222,222,222,222, +224,224,224,224,224,120,224,224,224,224,224,224,224,224,224,224, +224,224,224,224,224,224,224,224,120,120,120,120,120,120,120,120, +120,120,120,222,222,222,222,222,222,222,222,222,222,222,222,222, 222,222,216,222,222,222,222,222,222,222,222,222,222,222,222,222, 222,222,222,222,222,222,222,222,222,222,222,222,222,222,222,222, @@ -1850,399 +1904,399 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, 253,253,253,253,253,253,253,253,253,253,251,252,251,253,252,252, 252,251,251,251,251,251,251,251,251,252,252,252,252,251,252,252, -253,254,255,251,251,251,251,251,253,253,253,253,253,253,253,253, +253,254,255,113,113,251,251,251,253,253,253,253,253,253,253,253, 253,253,251,251,256,257,258,258,258,258,258,258,258,258,258,258, 259,260,253,253,253,253,253,253,253,253,253,253,253,253,253,253, /* block 19 */ -261,262,263,263,119,261,261,261,261,261,261,261,261,119,119,261, -261,119,119,261,261,261,261,261,261,261,261,261,261,261,261,261, -261,261,261,261,261,261,261,261,261,119,261,261,261,261,261,261, -261,119,261,119,119,119,261,261,261,261,119,119,262,261,264,263, -263,262,262,262,262,119,119,263,263,119,119,263,263,262,261,119, -119,119,119,119,119,119,119,264,119,119,119,119,261,261,119,261, -261,261,262,262,119,119,265,265,265,265,265,265,265,265,265,265, -261,261,266,266,267,267,267,267,267,267,268,266,261,269,262,119, +261,262,263,263,120,261,261,261,261,261,261,261,261,120,120,261, +261,120,120,261,261,261,261,261,261,261,261,261,261,261,261,261, +261,261,261,261,261,261,261,261,261,120,261,261,261,261,261,261, +261,120,261,120,120,120,261,261,261,261,120,120,262,261,264,263, +263,262,262,262,262,120,120,263,263,120,120,263,263,262,261,120, +120,120,120,120,120,120,120,264,120,120,120,120,261,261,120,261, +261,261,262,262,120,120,265,265,265,265,265,265,265,265,265,265, +261,261,266,266,267,267,267,267,267,267,268,266,261,269,262,120, /* block 20 */ -119,270,270,271,119,272,272,272,272,272,272,119,119,119,119,272, -272,119,119,272,272,272,272,272,272,272,272,272,272,272,272,272, -272,272,272,272,272,272,272,272,272,119,272,272,272,272,272,272, -272,119,272,272,119,272,272,119,272,272,119,119,270,119,271,271, -271,270,270,119,119,119,119,270,270,119,119,270,270,270,119,119, -119,270,119,119,119,119,119,119,119,272,272,272,272,119,272,119, -119,119,119,119,119,119,273,273,273,273,273,273,273,273,273,273, -270,270,272,272,272,270,274,119,119,119,119,119,119,119,119,119, +120,270,270,271,120,272,272,272,272,272,272,120,120,120,120,272, +272,120,120,272,272,272,272,272,272,272,272,272,272,272,272,272, +272,272,272,272,272,272,272,272,272,120,272,272,272,272,272,272, +272,120,272,272,120,272,272,120,272,272,120,120,270,120,271,271, +271,270,270,120,120,120,120,270,270,120,120,270,270,270,120,120, +120,270,120,120,120,120,120,120,120,272,272,272,272,120,272,120, +120,120,120,120,120,120,273,273,273,273,273,273,273,273,273,273, +270,270,272,272,272,270,274,120,120,120,120,120,120,120,120,120, /* block 21 */ -119,275,275,276,119,277,277,277,277,277,277,277,277,277,119,277, -277,277,119,277,277,277,277,277,277,277,277,277,277,277,277,277, -277,277,277,277,277,277,277,277,277,119,277,277,277,277,277,277, -277,119,277,277,119,277,277,277,277,277,119,119,275,277,276,276, -276,275,275,275,275,275,119,275,275,276,119,276,276,275,119,119, -277,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -277,277,275,275,119,119,278,278,278,278,278,278,278,278,278,278, -279,280,119,119,119,119,119,119,119,277,275,275,275,275,275,275, +120,275,275,276,120,277,277,277,277,277,277,277,277,277,120,277, +277,277,120,277,277,277,277,277,277,277,277,277,277,277,277,277, +277,277,277,277,277,277,277,277,277,120,277,277,277,277,277,277, +277,120,277,277,120,277,277,277,277,277,120,120,275,277,276,276, +276,275,275,275,275,275,120,275,275,276,120,276,276,275,120,120, +277,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +277,277,275,275,120,120,278,278,278,278,278,278,278,278,278,278, +279,280,120,120,120,120,120,120,120,277,275,275,275,275,275,275, /* block 22 */ -119,281,282,282,119,283,283,283,283,283,283,283,283,119,119,283, -283,119,119,283,283,283,283,283,283,283,283,283,283,283,283,283, -283,283,283,283,283,283,283,283,283,119,283,283,283,283,283,283, -283,119,283,283,119,283,283,283,283,283,119,119,281,283,284,281, -282,281,281,281,281,119,119,282,282,119,119,282,282,281,119,119, -119,119,119,119,119,119,281,284,119,119,119,119,283,283,119,283, -283,283,281,281,119,119,285,285,285,285,285,285,285,285,285,285, -286,283,287,287,287,287,287,287,119,119,119,119,119,119,119,119, +120,281,282,282,120,283,283,283,283,283,283,283,283,120,120,283, +283,120,120,283,283,283,283,283,283,283,283,283,283,283,283,283, +283,283,283,283,283,283,283,283,283,120,283,283,283,283,283,283, +283,120,283,283,120,283,283,283,283,283,120,120,281,283,284,281, +282,281,281,281,281,120,120,282,282,120,120,282,282,281,120,120, +120,120,120,120,120,281,281,284,120,120,120,120,283,283,120,283, +283,283,281,281,120,120,285,285,285,285,285,285,285,285,285,285, +286,283,287,287,287,287,287,287,120,120,120,120,120,120,120,120, /* block 23 */ -119,119,288,289,119,289,289,289,289,289,289,119,119,119,289,289, -289,119,289,289,289,289,119,119,119,289,289,119,289,119,289,289, -119,119,119,289,289,119,119,119,289,289,289,119,119,119,289,289, -289,289,289,289,289,289,289,289,289,289,119,119,119,119,290,291, -288,291,291,119,119,119,291,291,291,119,291,291,291,288,119,119, -289,119,119,119,119,119,119,290,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,292,292,292,292,292,292,292,292,292,292, -293,293,293,294,295,295,295,295,295,296,295,119,119,119,119,119, +120,120,288,289,120,289,289,289,289,289,289,120,120,120,289,289, +289,120,289,289,289,289,120,120,120,289,289,120,289,120,289,289, +120,120,120,289,289,120,120,120,289,289,289,120,120,120,289,289, +289,289,289,289,289,289,289,289,289,289,120,120,120,120,290,291, +288,291,291,120,120,120,291,291,291,120,291,291,291,288,120,120, +289,120,120,120,120,120,120,290,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,292,292,292,292,292,292,292,292,292,292, +293,293,293,294,295,295,295,295,295,296,295,120,120,120,120,120, /* block 24 */ -297,298,298,298,297,299,299,299,299,299,299,299,299,119,299,299, -299,119,299,299,299,299,299,299,299,299,299,299,299,299,299,299, -299,299,299,299,299,299,299,299,299,119,299,299,299,299,299,299, -299,299,299,299,299,299,299,299,299,299,119,119,119,299,297,297, -297,298,298,298,298,119,297,297,297,119,297,297,297,297,119,119, -119,119,119,119,119,297,297,119,299,299,299,119,119,119,119,119, -299,299,297,297,119,119,300,300,300,300,300,300,300,300,300,300, -119,119,119,119,119,119,119,119,301,301,301,301,301,301,301,302, +297,298,298,298,297,299,299,299,299,299,299,299,299,120,299,299, +299,120,299,299,299,299,299,299,299,299,299,299,299,299,299,299, +299,299,299,299,299,299,299,299,299,120,299,299,299,299,299,299, +299,299,299,299,299,299,299,299,299,299,120,120,120,299,297,297, +297,298,298,298,298,120,297,297,297,120,297,297,297,297,120,120, +120,120,120,120,120,297,297,120,299,299,299,120,120,120,120,120, +299,299,297,297,120,120,300,300,300,300,300,300,300,300,300,300, +120,120,120,120,120,120,120,301,302,302,302,302,302,302,302,303, /* block 25 */ -303,304,305,305,306,303,303,303,303,303,303,303,303,119,303,303, -303,119,303,303,303,303,303,303,303,303,303,303,303,303,303,303, -303,303,303,303,303,303,303,303,303,119,303,303,303,303,303,303, -303,303,303,303,119,303,303,303,303,303,119,119,304,303,305,304, -305,305,307,305,305,119,304,305,305,119,305,305,304,304,119,119, -119,119,119,119,119,307,307,119,119,119,119,119,119,119,303,119, -303,303,304,304,119,119,308,308,308,308,308,308,308,308,308,308, -119,303,303,119,119,119,119,119,119,119,119,119,119,119,119,119, +304,305,306,306,307,304,304,304,304,304,304,304,304,120,304,304, +304,120,304,304,304,304,304,304,304,304,304,304,304,304,304,304, +304,304,304,304,304,304,304,304,304,120,304,304,304,304,304,304, +304,304,304,304,120,304,304,304,304,304,120,120,305,304,306,305, +306,306,308,306,306,120,305,306,306,120,306,306,305,305,120,120, +120,120,120,120,120,308,308,120,120,120,120,120,120,120,304,120, +304,304,305,305,120,120,309,309,309,309,309,309,309,309,309,309, +120,304,304,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 26 */ -309,309,310,310,119,311,311,311,311,311,311,311,311,119,311,311, -311,119,311,311,311,311,311,311,311,311,311,311,311,311,311,311, -311,311,311,311,311,311,311,311,311,311,311,311,311,311,311,311, -311,311,311,311,311,311,311,311,311,311,311,309,309,311,312,310, -310,309,309,309,309,119,310,310,310,119,310,310,310,309,313,314, -119,119,119,119,311,311,311,312,315,315,315,315,315,315,315,311, -311,311,309,309,119,119,316,316,316,316,316,316,316,316,316,316, -315,315,315,315,315,315,315,315,315,314,311,311,311,311,311,311, +310,310,311,311,312,312,312,312,312,312,312,312,312,120,312,312, +312,120,312,312,312,312,312,312,312,312,312,312,312,312,312,312, +312,312,312,312,312,312,312,312,312,312,312,312,312,312,312,312, +312,312,312,312,312,312,312,312,312,312,312,310,310,312,313,311, +311,310,310,310,310,120,311,311,311,120,311,311,311,310,314,315, +120,120,120,120,312,312,312,313,316,316,316,316,316,316,316,312, +312,312,310,310,120,120,317,317,317,317,317,317,317,317,317,317, +316,316,316,316,316,316,316,316,316,315,312,312,312,312,312,312, /* block 27 */ -119,119,317,317,119,318,318,318,318,318,318,318,318,318,318,318, -318,318,318,318,318,318,318,119,119,119,318,318,318,318,318,318, -318,318,318,318,318,318,318,318,318,318,318,318,318,318,318,318, -318,318,119,318,318,318,318,318,318,318,318,318,119,318,119,119, -318,318,318,318,318,318,318,119,119,119,319,119,119,119,119,320, -317,317,319,319,319,119,319,119,317,317,317,317,317,317,317,320, -119,119,119,119,119,119,321,321,321,321,321,321,321,321,321,321, -119,119,317,317,322,119,119,119,119,119,119,119,119,119,119,119, +120,318,319,319,120,320,320,320,320,320,320,320,320,320,320,320, +320,320,320,320,320,320,320,120,120,120,320,320,320,320,320,320, +320,320,320,320,320,320,320,320,320,320,320,320,320,320,320,320, +320,320,120,320,320,320,320,320,320,320,320,320,120,320,120,120, +320,320,320,320,320,320,320,120,120,120,318,120,120,120,120,321, +319,319,318,318,318,120,318,120,319,319,319,319,319,319,319,321, +120,120,120,120,120,120,322,322,322,322,322,322,322,322,322,322, +120,120,319,319,323,120,120,120,120,120,120,120,120,120,120,120, /* block 28 */ -119,323,323,323,323,323,323,323,323,323,323,323,323,323,323,323, -323,323,323,323,323,323,323,323,323,323,323,323,323,323,323,323, -323,323,323,323,323,323,323,323,323,323,323,323,323,323,323,323, -323,324,323,325,324,324,324,324,324,324,324,119,119,119,119, 6, -323,323,323,323,323,323,326,324,324,324,324,324,324,324,324,327, -328,328,328,328,328,328,328,328,328,328,327,327,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324, +324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324, +324,324,324,324,324,324,324,324,324,324,324,324,324,324,324,324, +324,325,324,326,325,325,325,325,325,325,325,120,120,120,120, 6, +324,324,324,324,324,324,327,325,325,325,325,325,325,325,325,328, +329,329,329,329,329,329,329,329,329,329,328,328,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 29 */ -119,329,329,119,329,119,119,329,329,119,329,119,119,329,119,119, -119,119,119,119,329,329,329,329,119,329,329,329,329,329,329,329, -119,329,329,329,119,329,119,329,119,119,329,329,119,329,329,329, -329,330,329,331,330,330,330,330,330,330,119,330,330,329,119,119, -329,329,329,329,329,119,332,119,330,330,330,330,330,330,119,119, -333,333,333,333,333,333,333,333,333,333,119,119,329,329,329,329, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,330,330,120,330,120,330,330,330,330,330,120,330,330,330,330, +330,330,330,330,330,330,330,330,330,330,330,330,330,330,330,330, +330,330,330,330,120,330,120,330,330,330,330,330,330,330,330,330, +330,331,330,332,331,331,331,331,331,331,331,331,331,330,120,120, +330,330,330,330,330,120,333,120,331,331,331,331,331,331,120,120, +334,334,334,334,334,334,334,334,334,334,120,120,330,330,330,330, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 30 */ -334,335,335,335,336,336,336,336,336,336,336,336,336,336,336,336, -336,336,336,335,336,335,335,335,337,337,335,335,335,335,335,335, -338,338,338,338,338,338,338,338,338,338,339,339,339,339,339,339, -339,339,339,339,335,337,335,337,335,337,340,341,340,341,342,342, -334,334,334,334,334,334,334,334,119,334,334,334,334,334,334,334, -334,334,334,334,334,334,334,334,334,334,334,334,334,334,334,334, -334,334,334,334,334,334,334,334,334,334,334,334,334,119,119,119, -119,337,337,337,337,337,337,337,337,337,337,337,337,337,337,342, +335,336,336,336,337,337,337,337,337,337,337,337,337,337,337,337, +337,337,337,336,337,336,336,336,338,338,336,336,336,336,336,336, +339,339,339,339,339,339,339,339,339,339,340,340,340,340,340,340, +340,340,340,340,336,338,336,338,336,338,341,342,341,342,343,343, +335,335,335,335,335,335,335,335,120,335,335,335,335,335,335,335, +335,335,335,335,335,335,335,335,335,335,335,335,335,335,335,335, +335,335,335,335,335,335,335,335,335,335,335,335,335,120,120,120, +120,338,338,338,338,338,338,338,338,338,338,338,338,338,338,343, /* block 31 */ -337,337,337,337,337,336,337,337,334,334,334,334,334,337,337,337, -337,337,337,337,337,337,337,337,119,337,337,337,337,337,337,337, -337,337,337,337,337,337,337,337,337,337,337,337,337,337,337,337, -337,337,337,337,337,337,337,337,337,337,337,337,337,119,335,335, -335,335,335,335,335,335,337,335,335,335,335,335,335,119,335,335, -336,336,336,336,336, 20, 20, 20, 20,336,336,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +338,338,338,338,338,337,338,338,335,335,335,335,335,338,338,338, +338,338,338,338,338,338,338,338,120,338,338,338,338,338,338,338, +338,338,338,338,338,338,338,338,338,338,338,338,338,338,338,338, +338,338,338,338,338,338,338,338,338,338,338,338,338,120,336,336, +336,336,336,336,336,336,338,336,336,336,336,336,336,120,336,336, +337,337,337,337,337, 20, 20, 20, 20,337,337,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 32 */ -343,343,343,343,343,343,343,343,343,343,343,343,343,343,343,343, -343,343,343,343,343,343,343,343,343,343,343,343,343,343,343,343, -343,343,343,343,343,343,343,343,343,343,343,344,344,345,345,345, -345,346,345,345,345,345,345,345,344,345,345,346,346,345,345,343, -347,347,347,347,347,347,347,347,347,347,348,348,348,348,348,348, -343,343,343,343,343,343,346,346,345,345,343,343,343,343,345,345, -345,343,344,344,344,343,343,344,344,344,344,344,344,344,343,343, -343,345,345,345,345,343,343,343,343,343,343,343,343,343,343,343, +344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344, +344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344, +344,344,344,344,344,344,344,344,344,344,344,345,345,346,346,346, +346,347,346,346,346,346,346,346,345,346,346,347,347,346,346,344, +348,348,348,348,348,348,348,348,348,348,349,349,349,349,349,349, +344,344,344,344,344,344,347,347,346,346,344,344,344,344,346,346, +346,344,345,345,345,344,344,345,345,345,345,345,345,345,344,344, +344,346,346,346,346,344,344,344,344,344,344,344,344,344,344,344, /* block 33 */ -343,343,345,344,346,345,345,344,344,344,344,344,344,345,343,344, -349,349,349,349,349,349,349,349,349,349,344,344,344,345,350,350, -351,351,351,351,351,351,351,351,351,351,351,351,351,351,351,351, -351,351,351,351,351,351,351,351,351,351,351,351,351,351,351,351, -351,351,351,351,351,351,119,351,119,119,119,119,119,351,119,119, +344,344,346,345,347,346,346,345,345,345,345,345,345,346,344,345, +350,350,350,350,350,350,350,350,350,350,345,345,345,346,351,351, 352,352,352,352,352,352,352,352,352,352,352,352,352,352,352,352, 352,352,352,352,352,352,352,352,352,352,352,352,352,352,352,352, -352,352,352,352,352,352,352,352,352,352,352,353,354,352,352,352, +352,352,352,352,352,352,120,352,120,120,120,120,120,352,120,120, +353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353, +353,353,353,353,353,353,353,353,353,353,353,353,353,353,353,353, +353,353,353,353,353,353,353,353,353,353,353,354,355,353,353,353, /* block 34 */ -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, 356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, 356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, - -/* block 35 */ 356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, 356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, -356,356,356,356,356,356,356,356,357,357,357,357,357,357,357,357, -357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, +356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, +356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, 357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, 357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, + +/* block 35 */ 357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, 357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, - -/* block 36 */ -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, +357,357,357,357,357,357,357,357,358,358,358,358,358,358,358,358, 358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, 358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, 358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,119,358,358,358,358,119,119, -358,358,358,358,358,358,358,119,358,119,358,358,358,358,119,119, 358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, 358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, +/* block 36 */ +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,120,359,359,359,359,120,120, +359,359,359,359,359,359,359,120,359,120,359,359,359,359,120,120, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, + /* block 37 */ -358,358,358,358,358,358,358,358,358,119,358,358,358,358,119,119, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,119,358,358,358,358,119,119,358,358,358,358,358,358,358,119, -358,119,358,358,358,358,119,119,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, +359,359,359,359,359,359,359,359,359,120,359,359,359,359,120,120, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,120,359,359,359,359,120,120,359,359,359,359,359,359,359,120, +359,120,359,359,359,359,120,120,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, /* block 38 */ -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,119,358,358,358,358,119,119,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,358,358,358,358,119,119,359,359,359, -360,360,360,360,360,360,360,360,360,361,361,361,361,361,361,361, -361,361,361,361,361,361,361,361,361,361,361,361,361,119,119,119, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,120,359,359,359,359,120,120,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,359,359,359,359,120,120,360,360,360, +361,361,361,361,361,361,361,361,361,362,362,362,362,362,362,362, +362,362,362,362,362,362,362,362,362,362,362,362,362,120,120,120, /* block 39 */ -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -362,362,362,362,362,362,362,362,362,362,119,119,119,119,119,119, -363,363,363,363,363,363,363,363,363,363,363,363,363,363,363,363, -363,363,363,363,363,363,363,363,363,363,363,363,363,363,363,363, -363,363,363,363,363,363,363,363,363,363,363,363,363,363,363,363, -363,363,363,363,363,363,363,363,363,363,363,363,363,363,363,363, -363,363,363,363,363,363,363,363,363,363,363,363,363,363,363,363, -364,364,364,364,364,364,119,119,365,365,365,365,365,365,119,119, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +363,363,363,363,363,363,363,363,363,363,120,120,120,120,120,120, +364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364, +364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364, +364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364, +364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364, +364,364,364,364,364,364,364,364,364,364,364,364,364,364,364,364, +365,365,365,365,365,365,120,120,366,366,366,366,366,366,120,120, /* block 40 */ -366,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, +367,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, /* block 41 */ -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, /* block 42 */ -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,368,368,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,369,370,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, /* block 43 */ -369,370,370,370,370,370,370,370,370,370,370,370,370,370,370,370, -370,370,370,370,370,370,370,370,370,370,370,371,372,119,119,119, -373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373, -373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373, -373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373, -373,373,373,373,373,373,373,373,373,373,373,373,373,373,373,373, -373,373,373,373,373,373,373,373,373,373,373, 5, 5, 5,374,374, -374,373,373,373,373,373,373,373,373,119,119,119,119,119,119,119, +371,372,372,372,372,372,372,372,372,372,372,372,372,372,372,372, +372,372,372,372,372,372,372,372,372,372,372,373,374,120,120,120, +375,375,375,375,375,375,375,375,375,375,375,375,375,375,375,375, +375,375,375,375,375,375,375,375,375,375,375,375,375,375,375,375, +375,375,375,375,375,375,375,375,375,375,375,375,375,375,375,375, +375,375,375,375,375,375,375,375,375,375,375,375,375,375,375,375, +375,375,375,375,375,375,375,375,375,375,375, 5, 5, 5,376,376, +376,375,375,375,375,375,375,375,375,120,120,120,120,120,120,120, /* block 44 */ -375,375,375,375,375,375,375,375,375,375,375,375,375,119,375,375, -375,375,376,376,376,119,119,119,119,119,119,119,119,119,119,119, -377,377,377,377,377,377,377,377,377,377,377,377,377,377,377,377, -377,377,378,378,378,379,379,119,119,119,119,119,119,119,119,119, -380,380,380,380,380,380,380,380,380,380,380,380,380,380,380,380, -380,380,381,381,119,119,119,119,119,119,119,119,119,119,119,119, -382,382,382,382,382,382,382,382,382,382,382,382,382,119,382,382, -382,119,383,383,119,119,119,119,119,119,119,119,119,119,119,119, +377,377,377,377,377,377,377,377,377,377,377,377,377,120,377,377, +377,377,378,378,378,120,120,120,120,120,120,120,120,120,120,120, +379,379,379,379,379,379,379,379,379,379,379,379,379,379,379,379, +379,379,380,380,380,381,381,120,120,120,120,120,120,120,120,120, +382,382,382,382,382,382,382,382,382,382,382,382,382,382,382,382, +382,382,383,383,120,120,120,120,120,120,120,120,120,120,120,120, +384,384,384,384,384,384,384,384,384,384,384,384,384,120,384,384, +384,120,385,385,120,120,120,120,120,120,120,120,120,120,120,120, /* block 45 */ -384,384,384,384,384,384,384,384,384,384,384,384,384,384,384,384, -384,384,384,384,384,384,384,384,384,384,384,384,384,384,384,384, -384,384,384,384,384,384,384,384,384,384,384,384,384,384,384,384, -384,384,384,384,385,385,386,385,385,385,385,385,385,385,386,386, -386,386,386,386,386,386,385,386,386,385,385,385,385,385,385,385, -385,385,385,385,387,387,387,388,387,387,387,389,384,385,119,119, -390,390,390,390,390,390,390,390,390,390,119,119,119,119,119,119, -391,391,391,391,391,391,391,391,391,391,119,119,119,119,119,119, +386,386,386,386,386,386,386,386,386,386,386,386,386,386,386,386, +386,386,386,386,386,386,386,386,386,386,386,386,386,386,386,386, +386,386,386,386,386,386,386,386,386,386,386,386,386,386,386,386, +386,386,386,386,387,387,388,387,387,387,387,387,387,387,388,388, +388,388,388,388,388,388,387,388,388,387,387,387,387,387,387,387, +387,387,387,387,389,389,389,390,389,389,389,391,386,387,120,120, +392,392,392,392,392,392,392,392,392,392,120,120,120,120,120,120, +393,393,393,393,393,393,393,393,393,393,120,120,120,120,120,120, /* block 46 */ -392,392,393,393,392,393,394,392,392,392,392,395,395,395,396,119, -397,397,397,397,397,397,397,397,397,397,119,119,119,119,119,119, -398,398,398,398,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,399,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,119,119,119,119,119,119,119, +394,394,395,395,394,395,396,394,394,394,394,397,397,397,398,120, +399,399,399,399,399,399,399,399,399,399,120,120,120,120,120,120, +400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,401,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,120,120,120,120,120,120,120, /* block 47 */ -398,398,398,398,398,395,395,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,398,398,398,398,398,398,398, -398,398,398,398,398,398,398,398,398,395,398,119,119,119,119,119, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,367,367,367,367,367,367,367,367,367,367, -367,367,367,367,367,367,119,119,119,119,119,119,119,119,119,119, +400,400,400,400,400,397,397,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, +400,400,400,400,400,400,400,400,400,397,400,120,120,120,120,120, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,368,368,368,368,368,368,368,368,368,368, +368,368,368,368,368,368,120,120,120,120,120,120,120,120,120,120, /* block 48 */ -400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,400, -400,400,400,400,400,400,400,400,400,400,400,400,400,400,400,119, -401,401,401,402,402,402,402,401,401,402,402,402,119,119,119,119, -402,402,401,402,402,402,402,402,402,401,401,401,119,119,119,119, -403,119,119,119,404,404,405,405,405,405,405,405,405,405,405,405, -406,406,406,406,406,406,406,406,406,406,406,406,406,406,406,406, -406,406,406,406,406,406,406,406,406,406,406,406,406,406,119,119, -406,406,406,406,406,119,119,119,119,119,119,119,119,119,119,119, +402,402,402,402,402,402,402,402,402,402,402,402,402,402,402,402, +402,402,402,402,402,402,402,402,402,402,402,402,402,402,402,120, +403,403,403,404,404,404,404,403,403,404,404,404,120,120,120,120, +404,404,403,404,404,404,404,404,404,403,403,403,120,120,120,120, +405,120,120,120,406,406,407,407,407,407,407,407,407,407,407,407, +408,408,408,408,408,408,408,408,408,408,408,408,408,408,408,408, +408,408,408,408,408,408,408,408,408,408,408,408,408,408,120,120, +408,408,408,408,408,120,120,120,120,120,120,120,120,120,120,120, /* block 49 */ -407,407,407,407,407,407,407,407,407,407,407,407,407,407,407,407, -407,407,407,407,407,407,407,407,407,407,407,407,407,407,407,407, -407,407,407,407,407,407,407,407,407,407,407,407,119,119,119,119, -407,407,407,407,407,407,407,407,407,407,407,407,407,407,407,407, -407,407,407,407,407,407,407,407,407,407,119,119,119,119,119,119, -408,408,408,408,408,408,408,408,408,408,409,119,119,119,410,410, -411,411,411,411,411,411,411,411,411,411,411,411,411,411,411,411, -411,411,411,411,411,411,411,411,411,411,411,411,411,411,411,411, +409,409,409,409,409,409,409,409,409,409,409,409,409,409,409,409, +409,409,409,409,409,409,409,409,409,409,409,409,409,409,409,409, +409,409,409,409,409,409,409,409,409,409,409,409,120,120,120,120, +409,409,409,409,409,409,409,409,409,409,409,409,409,409,409,409, +409,409,409,409,409,409,409,409,409,409,120,120,120,120,120,120, +410,410,410,410,410,410,410,410,410,410,411,120,120,120,412,412, +413,413,413,413,413,413,413,413,413,413,413,413,413,413,413,413, +413,413,413,413,413,413,413,413,413,413,413,413,413,413,413,413, /* block 50 */ -412,412,412,412,412,412,412,412,412,412,412,412,412,412,412,412, -412,412,412,412,412,412,412,413,413,414,414,413,119,119,415,415, -416,416,416,416,416,416,416,416,416,416,416,416,416,416,416,416, -416,416,416,416,416,416,416,416,416,416,416,416,416,416,416,416, -416,416,416,416,416,416,416,416,416,416,416,416,416,416,416,416, -416,416,416,416,416,417,418,417,418,418,418,418,418,418,418,119, -418,419,418,419,419,418,418,418,418,418,418,418,418,417,417,417, -417,417,417,418,418,418,418,418,418,418,418,418,418,119,119,418, +414,414,414,414,414,414,414,414,414,414,414,414,414,414,414,414, +414,414,414,414,414,414,414,415,415,416,416,415,120,120,417,417, +418,418,418,418,418,418,418,418,418,418,418,418,418,418,418,418, +418,418,418,418,418,418,418,418,418,418,418,418,418,418,418,418, +418,418,418,418,418,418,418,418,418,418,418,418,418,418,418,418, +418,418,418,418,418,419,420,419,420,420,420,420,420,420,420,120, +420,421,420,421,421,420,420,420,420,420,420,420,420,419,419,419, +419,419,419,420,420,420,420,420,420,420,420,420,420,120,120,420, /* block 51 */ -420,420,420,420,420,420,420,420,420,420,119,119,119,119,119,119, -420,420,420,420,420,420,420,420,420,420,119,119,119,119,119,119, -421,421,421,421,421,421,421,422,421,421,421,421,421,421,119,119, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,423,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +422,422,422,422,422,422,422,422,422,422,120,120,120,120,120,120, +422,422,422,422,422,422,422,422,422,422,120,120,120,120,120,120, +423,423,423,423,423,423,423,424,423,423,423,423,423,423,120,120, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,425,113, +113,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 52 */ -424,424,424,424,425,426,426,426,426,426,426,426,426,426,426,426, -426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426, -426,426,426,426,426,426,426,426,426,426,426,426,426,426,426,426, -426,426,426,426,424,425,424,424,424,424,424,425,424,425,425,425, -425,425,424,425,425,426,426,426,426,426,426,426,119,119,119,119, -427,427,427,427,427,427,427,427,427,427,428,428,428,428,428,428, -428,429,429,429,429,429,429,429,429,429,429,424,424,424,424,424, -424,424,424,424,429,429,429,429,429,429,429,429,429,119,119,119, +426,426,426,426,427,428,428,428,428,428,428,428,428,428,428,428, +428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428, +428,428,428,428,428,428,428,428,428,428,428,428,428,428,428,428, +428,428,428,428,426,429,426,426,426,426,426,427,426,427,427,427, +427,427,426,427,427,428,428,428,428,428,428,428,120,120,120,120, +430,430,430,430,430,430,430,430,430,430,431,431,431,431,431,431, +431,432,432,432,432,432,432,432,432,432,432,426,426,426,426,426, +426,426,426,426,432,432,432,432,432,432,432,432,432,120,120,120, /* block 53 */ -430,430,431,432,432,432,432,432,432,432,432,432,432,432,432,432, -432,432,432,432,432,432,432,432,432,432,432,432,432,432,432,432, -432,431,430,430,430,430,431,431,430,430,431,430,430,430,432,432, -433,433,433,433,433,433,433,433,433,433,432,432,432,432,432,432, -434,434,434,434,434,434,434,434,434,434,434,434,434,434,434,434, -434,434,434,434,434,434,434,434,434,434,434,434,434,434,434,434, -434,434,434,434,434,434,435,436,435,435,436,436,436,435,436,435, -435,435,436,436,119,119,119,119,119,119,119,119,437,437,437,437, +433,433,434,435,435,435,435,435,435,435,435,435,435,435,435,435, +435,435,435,435,435,435,435,435,435,435,435,435,435,435,435,435, +435,434,433,433,433,433,434,434,433,433,434,433,433,433,435,435, +436,436,436,436,436,436,436,436,436,436,435,435,435,435,435,435, +437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437, +437,437,437,437,437,437,437,437,437,437,437,437,437,437,437,437, +437,437,437,437,437,437,438,439,438,438,439,439,439,438,439,438, +438,438,439,439,120,120,120,120,120,120,120,120,440,440,440,440, /* block 54 */ -438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438, -438,438,438,438,438,438,438,438,438,438,438,438,438,438,438,438, -438,438,438,438,439,439,439,439,439,439,439,439,440,440,440,440, -440,440,440,440,439,439,440,440,119,119,119,441,441,441,441,441, -442,442,442,442,442,442,442,442,442,442,119,119,119,438,438,438, -443,443,443,443,443,443,443,443,443,443,444,444,444,444,444,444, -444,444,444,444,444,444,444,444,444,444,444,444,444,444,444,444, -444,444,444,444,444,444,444,444,445,445,445,445,445,445,446,446, +441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441, +441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441, +441,441,441,441,442,442,442,442,442,442,442,442,443,443,443,443, +443,443,443,443,442,442,443,443,120,120,120,444,444,444,444,444, +445,445,445,445,445,445,445,445,445,445,120,120,120,441,441,441, +446,446,446,446,446,446,446,446,446,446,447,447,447,447,447,447, +447,447,447,447,447,447,447,447,447,447,447,447,447,447,447,447, +447,447,447,447,447,447,447,447,448,448,448,448,448,448,449,449, /* block 55 */ -447,448,449,450,451,452,453,454,455,119,119,119,119,119,119,119, -456,456,456,456,456,456,456,456,456,456,456,456,456,456,456,456, -456,456,456,456,456,456,456,456,456,456,456,456,456,456,456,456, -456,456,456,456,456,456,456,456,456,456,456,119,119,456,456,456, -457,457,457,457,457,457,457,457,119,119,119,119,119,119,119,119, -458,459,458,460,459,461,461,462,461,462,463,459,462,462,459,459, -462,464,459,459,459,459,459,459,459,465,466,465,465,461,465,465, -465,465,467,467,468,466,466,469,470,470,119,119,119,119,119,119, +450,451,452,453,454,455,456,457,458,120,120,120,120,120,120,120, +459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,459, +459,459,459,459,459,459,459,459,459,459,459,459,459,459,459,459, +459,459,459,459,459,459,459,459,459,459,459,120,120,459,459,459, +460,460,460,460,460,460,460,460,120,120,120,120,120,120,120,120, +461,462,461,463,462,464,464,465,464,465,466,462,465,465,462,462, +465,467,462,462,462,462,462,462,462,468,469,470,470,464,470,470, +470,470,471,472,473,469,469,474,475,475,476,120,120,120,120,120, /* block 56 */ 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35,127,127,127,127,127,471,109,109,109,109, -109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109, -109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109, -109,109,109,109,109,109,109,109,109,109,109,109,109,120,120,120, -120,120,109,109,109,109,120,120,120,120,120, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35,472,473, 35, 35, 35,474, 35, 35, + 35, 35, 35, 35, 35, 35,128,128,128,128,128,477,110,110,110,110, +110,110,110,110,110,110,110,110,110,110,110,110,110,110,110,110, +110,110,110,110,110,110,110,110,110,110,110,110,110,110,110,110, +110,110,110,110,110,110,110,110,110,110,110,110,110,121,121,121, +121,121,110,110,110,110,121,121,121,121,121, 35, 35, 35, 35, 35, + 35, 35, 35, 35, 35, 35, 35, 35,478,479, 35, 35, 35,480, 35, 35, /* block 57 */ - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,109,109,109,109,109, -109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,109, -109,109,109,109,109,109,109,109,109,109,109,109,109,109,109,120, -113,113,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,119,112,112,112,112,112, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,481, 35, + 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35,110,110,110,110,110, +110,110,110,110,110,110,110,110,110,110,110,110,110,110,110,110, +110,110,110,110,110,110,110,110,110,110,110,110,110,110,110,121, +114,114,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,482,113,120,113,113,113,113,113, /* block 58 */ 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, @@ -2251,12 +2305,12 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, -475,476, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, +483,484, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, /* block 59 */ 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, - 32, 33, 32, 33, 32, 33, 35, 35, 35, 35, 35,477, 35, 35,478, 35, + 32, 33, 32, 33, 32, 33, 35, 35, 35, 35, 35,485, 35, 35,486, 35, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, @@ -2265,58 +2319,58 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, /* block 60 */ -479,479,479,479,479,479,479,479,480,480,480,480,480,480,480,480, -479,479,479,479,479,479,119,119,480,480,480,480,480,480,119,119, -479,479,479,479,479,479,479,479,480,480,480,480,480,480,480,480, -479,479,479,479,479,479,479,479,480,480,480,480,480,480,480,480, -479,479,479,479,479,479,119,119,480,480,480,480,480,480,119,119, -127,479,127,479,127,479,127,479,119,480,119,480,119,480,119,480, -479,479,479,479,479,479,479,479,480,480,480,480,480,480,480,480, -481,481,482,482,482,482,483,483,484,484,485,485,486,486,119,119, +487,487,487,487,487,487,487,487,488,488,488,488,488,488,488,488, +487,487,487,487,487,487,120,120,488,488,488,488,488,488,120,120, +487,487,487,487,487,487,487,487,488,488,488,488,488,488,488,488, +487,487,487,487,487,487,487,487,488,488,488,488,488,488,488,488, +487,487,487,487,487,487,120,120,488,488,488,488,488,488,120,120, +128,487,128,487,128,487,128,487,120,488,120,488,120,488,120,488, +487,487,487,487,487,487,487,487,488,488,488,488,488,488,488,488, +489,489,490,490,490,490,491,491,492,492,493,493,494,494,120,120, /* block 61 */ -479,479,479,479,479,479,479,479,487,487,487,487,487,487,487,487, -479,479,479,479,479,479,479,479,487,487,487,487,487,487,487,487, -479,479,479,479,479,479,479,479,487,487,487,487,487,487,487,487, -479,479,127,488,127,119,127,127,480,480,489,489,490,118,491,118, -118,118,127,488,127,119,127,127,492,492,492,492,490,118,118,118, -479,479,127,127,119,119,127,127,480,480,493,493,119,118,118,118, -479,479,127,127,127,168,127,127,480,480,494,494,173,118,118,118, -119,119,127,488,127,119,127,127,495,495,496,496,490,118,118,119, +487,487,487,487,487,487,487,487,495,495,495,495,495,495,495,495, +487,487,487,487,487,487,487,487,495,495,495,495,495,495,495,495, +487,487,487,487,487,487,487,487,495,495,495,495,495,495,495,495, +487,487,128,496,128,120,128,128,488,488,497,497,498,119,499,119, +119,119,128,496,128,120,128,128,500,500,500,500,498,119,119,119, +487,487,128,128,120,120,128,128,488,488,501,501,120,119,119,119, +487,487,128,128,128,169,128,128,488,488,502,502,174,119,119,119, +120,120,128,496,128,120,128,128,503,503,504,504,498,119,119,120, /* block 62 */ - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24,497,498, 24, 24, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 24,505,506, 24, 24, 10, 10, 10, 10, 10, 10, 5, 5, 23, 27, 7, 23, 23, 27, 7, 23, - 5, 5, 5, 5, 5, 5, 5, 5,499,500, 24, 24, 24, 24, 24, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 23, 27, 5,501, 5, 5, 16, - 16, 5, 5, 5, 9, 7, 8, 5, 5,501, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5,507,508, 24, 24, 24, 24, 24,509, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 23, 27, 5,510, 5, 5, 16, + 16, 5, 5, 5, 9, 7, 8, 5, 5,510, 5, 5, 5, 5, 5, 5, 5, 5, 9, 5, 16, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, - 24, 24, 24, 24, 24,502, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 25,109,119,119, 25, 25, 25, 25, 25, 25, 9, 9, 9, 7, 8,109, + 24, 24, 24, 24, 24,511, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 25,110,120,120, 25, 25, 25, 25, 25, 25, 9, 9, 9, 7, 8,110, /* block 63 */ - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 9, 9, 9, 7, 8,119, -109,109,109,109,109,109,109,109,109,109,109,109,109,119,119,119, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 9, 9, 9, 7, 8,120, +110,110,110,110,110,110,110,110,110,110,110,110,110,120,120,120, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -112,112,112,112,112,112,112,112,112,112,112,112,112,423,423,423, -423,112,423,423,423,112,112,112,112,112,112,112,112,112,112,112, -503,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +113,113,113,113,113,113,113,113,113,113,113,113,113,425,425,425, +425,113,425,425,425,113,113,113,113,113,113,113,113,113,113,113, +512,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 64 */ - 20, 20,504, 20, 20, 20, 20,504, 20, 20,505,504,504,504,505,505, -504,504,504,505, 20,504, 20, 20, 9,504,504,504,504,504, 20, 20, - 20, 20, 21, 20,504, 20,506, 20,504, 20,507,508,504,504, 20,505, -504,504,509,504,505,510,510,510,510,511, 20, 20,505,505,504,504, - 9, 9, 9, 9, 9,504,505,505,505,505, 20, 9, 20, 20,512, 20, + 20, 20,513, 20, 20, 20, 20,513, 20, 20,514,513,513,513,514,514, +513,513,513,514, 20,513, 20, 20, 9,513,513,513,513,513, 20, 20, + 20, 20, 21, 20,513, 20,515, 20,513, 20,516,517,513,513, 20,514, +513,513,518,513,514,519,519,519,519,520, 20, 20,514,514,513,513, + 9, 9, 9, 9, 9,513,514,514,514,514, 20, 9, 20, 20,521, 20, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, -513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, -514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +522,522,522,522,522,522,522,522,522,522,522,522,522,522,522,522, +523,523,523,523,523,523,523,523,523,523,523,523,523,523,523,523, /* block 65 */ -515,515,515, 32, 33,515,515,515,515, 25, 20, 20,119,119,119,119, - 9, 9, 9, 9,516, 21, 21, 21, 21, 21, 9, 9, 20, 20, 20, 20, +524,524,524, 32, 33,524,524,524,524, 25, 20, 20,120,120,120,120, + 9, 9, 9, 9,525, 21, 21, 21, 21, 21, 9, 9, 20, 20, 20, 20, 9, 20, 20, 9, 20, 20, 9, 20, 20, 21, 21, 20, 20, 20, 9, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 9, 9, @@ -2357,10 +2411,10 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ /* block 69 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, + 20, 20, 20, 20, 20, 20, 20,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, @@ -2368,10 +2422,10 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20,517,517,517,517,517,517,517,517,517,517, -517,517,518,517,517,517,517,517,517,517,517,517,517,517,517,517, -519,519,519,519,519,519,519,519,519,519,519,519,519,519,519,519, -519,519,519,519,519,519,519,519,519,519, 25, 25, 25, 25, 25, 25, + 20, 20, 20, 20, 20, 20,526,526,526,526,526,526,526,526,526,526, +526,526,527,526,526,526,526,526,526,526,526,526,526,526,526,526, +528,528,528,528,528,528,528,528,528,528,528,528,528,528,528,528, +528,528,528,528,528,528,528,528,528,528, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, /* block 71 */ @@ -2392,7 +2446,7 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 21, 9, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 9, 9, 9,516,516,516,516, 9, + 20, 20, 20, 20, 20, 20, 20, 20, 9, 9, 9,525,525,525,525, 9, /* block 73 */ 21, 21, 21, 21, 21, 21, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -2401,7 +2455,7 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,516, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,525, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, /* block 74 */ @@ -2435,20 +2489,20 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, /* block 77 */ -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, -520,520,520,520,520,520,520,520,520,520,520,520,520,520,520,520, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, +529,529,529,529,529,529,529,529,529,529,529,529,529,529,529,529, /* block 78 */ 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, - 9, 9, 9, 9,516,516, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9,525,525, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -2472,167 +2526,167 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 9, 9, 9, 9, 9, 20, 20, 9, 9, 9, 9, 9, 9, 20, 20, 20, 21, 20, 20, 20, 20, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20,119,119, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20,120,120, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, /* block 81 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20,119,119, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20,120, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20,119, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,119, /* block 82 */ -521,521,521,521,521,521,521,521,521,521,521,521,521,521,521,521, -521,521,521,521,521,521,521,521,521,521,521,521,521,521,521,521, -521,521,521,521,521,521,521,521,521,521,521,521,521,521,521,119, -522,522,522,522,522,522,522,522,522,522,522,522,522,522,522,522, -522,522,522,522,522,522,522,522,522,522,522,522,522,522,522,522, -522,522,522,522,522,522,522,522,522,522,522,522,522,522,522,119, - 32, 33,523,524,525,526,527, 32, 33, 32, 33, 32, 33,528,529,530, -531, 35, 32, 33, 35, 32, 33, 35, 35, 35, 35, 35,109,109,532,532, +530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530, +530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,530, +530,530,530,530,530,530,530,530,530,530,530,530,530,530,530,120, +531,531,531,531,531,531,531,531,531,531,531,531,531,531,531,531, +531,531,531,531,531,531,531,531,531,531,531,531,531,531,531,531, +531,531,531,531,531,531,531,531,531,531,531,531,531,531,531,120, + 32, 33,532,533,534,535,536, 32, 33, 32, 33, 32, 33,537,538,539, +540, 35, 32, 33, 35, 32, 33, 35, 35, 35, 35, 35,110,110,541,541, /* block 83 */ -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,164,165,164,165,164,165,164,165,164,165,164,165, -164,165,164,165,533,534,534,534,534,534,534,164,165,164,165,535, -535,535,164,165,119,119,119,119,119,536,536,536,536,537,536,536, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,165,166,165,166,165,166,165,166,165,166,165,166, +165,166,165,166,542,543,543,543,543,543,543,165,166,165,166,544, +544,544,165,166,120,120,120,120,120,545,545,545,545,546,545,545, /* block 84 */ -538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538, -538,538,538,538,538,538,538,538,538,538,538,538,538,538,538,538, -538,538,538,538,538,538,119,538,119,119,119,119,119,538,119,119, -539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539, -539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539, -539,539,539,539,539,539,539,539,539,539,539,539,539,539,539,539, -539,539,539,539,539,539,539,539,119,119,119,119,119,119,119,540, -541,119,119,119,119,119,119,119,119,119,119,119,119,119,119,542, +547,547,547,547,547,547,547,547,547,547,547,547,547,547,547,547, +547,547,547,547,547,547,547,547,547,547,547,547,547,547,547,547, +547,547,547,547,547,547,120,547,120,120,120,120,120,547,120,120, +548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548, +548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548, +548,548,548,548,548,548,548,548,548,548,548,548,548,548,548,548, +548,548,548,548,548,548,548,548,120,120,120,120,120,120,120,549, +550,120,120,120,120,120,120,120,120,120,120,120,120,120,120,551, /* block 85 */ -358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, -358,358,358,358,358,358,358,119,119,119,119,119,119,119,119,119, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,119, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,119, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,119, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,119, -543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543, -543,543,543,543,543,543,543,543,543,543,543,543,543,543,543,543, +359,359,359,359,359,359,359,359,359,359,359,359,359,359,359,359, +359,359,359,359,359,359,359,120,120,120,120,120,120,120,120,120, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,120, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,120, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,120, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,120, +552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552, +552,552,552,552,552,552,552,552,552,552,552,552,552,552,552,552, /* block 86 */ 5, 5, 23, 27, 23, 27, 5, 5, 5, 23, 27, 5, 23, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 10, 5, 5, 10, 5, 23, 27, 5, 5, - 23, 27, 7, 8, 7, 8, 7, 8, 7, 8, 5, 5, 5, 5, 5,110, + 23, 27, 7, 8, 7, 8, 7, 8, 7, 8, 5, 5, 5, 5, 5,111, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 10, 10, 5, 5, 5, 5, - 10, 5, 7,544, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, + 10, 5, 7,553, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 20, 20, 5,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 87 */ -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,119,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,119,119,119,119,119,119,119,119,119,119,119,119, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,120,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,120,120,120,120,120,120,120,120,120,120,120,120, /* block 88 */ -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, /* block 89 */ -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,545,545,545,545,545,545,545,545,545,545, -545,545,545,545,545,545,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,119,119,119,119, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,554,554,554,554,554,554,554,554,554,554, +554,554,554,554,554,554,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,120,120,120,120, /* block 90 */ - 4,546,546,547, 20,548,549,550,551,552,551,552,551,552,551,552, -551,552, 20,553,551,552,551,552,551,552,551,552,554,555,556,556, - 20,550,550,550,550,550,550,550,550,550,557,557,557,557,558,558, -559,560,560,560,560,560, 20,553,550,550,550,548,561,562,563,563, -119,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, + 4,555,555,556, 20,557,558,559,560,561,560,561,560,561,560,561, +560,561, 20,562,560,561,560,561,560,561,560,561,563,564,565,565, + 20,559,559,559,559,559,559,559,559,559,566,566,566,566,567,567, +568,569,569,569,569,569, 20,562,559,559,559,557,570,571,572,572, +120,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, /* block 91 */ -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,119,119,565,565,566,566,567,567,564, -568,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,546,560,570,570,569, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,120,120,574,574,575,575,576,576,573, +577,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,555,569,579,579,578, /* block 92 */ -119,119,119,119,119,571,571,571,571,571,571,571,571,571,571,571, -571,571,571,571,571,571,571,571,571,571,571,571,571,571,571,571, -571,571,571,571,571,571,571,571,571,571,571,571,571,571,571,571, -119,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +120,120,120,120,120,580,580,580,580,580,580,580,580,580,580,580, +580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, +580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, +120,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, /* block 93 */ -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,119, -563,563,573,573,573,573,563,563,563,563,563,563,563,563,563,563, -571,571,571,571,571,571,571,571,571,571,571,571,571,571,571,571, -571,571,571,571,571,571,571,571,571,571,571,119,119,119,119,119, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563,563,563,563,119,119,119,119,119,119,119,119,119,119,119,119, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,120, +572,572,582,582,582,582,572,572,572,572,572,572,572,572,572,572, +580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, +580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572,572,572,572,120,120,120,120,120,120,120,120,120,120,120,120, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, /* block 94 */ -574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,574, -574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,119, -573,573,573,573,573,573,573,573,573,573,563,563,563,563,563,563, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563,563,563,563,563,563,563,563, 25, 25, 25, 25, 25, 25, 25, 25, +583,583,583,583,583,583,583,583,583,583,583,583,583,583,583,583, +583,583,583,583,583,583,583,583,583,583,583,583,583,583,583,120, +582,582,582,582,582,582,582,582,582,582,572,572,572,572,572,572, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572,572,572,572,572,572,572,572, 25, 25, 25, 25, 25, 25, 25, 25, 20, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, -574,574,574,574,574,574,574,574,574,574,574,574,574,574,574,574, -574,574,574,574,574,574,574,574,574,574,574,574,574,574,574, 20, +583,583,583,583,583,583,583,583,583,583,583,583,583,583,583,583, +583,583,583,583,583,583,583,583,583,583,583,583,583,583,583, 20, /* block 95 */ -573,573,573,573,573,573,573,573,573,573,563,563,563,563,563,563, -563,563,563,563,563,563,563,575,563,575,563,563,563,563,563,563, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, -563,563,563,563,563,563,563,563,563,563,563,563, 20, 20, 20, 20, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,119, +582,582,582,582,582,582,582,582,582,582,572,572,572,572,572,572, +572,572,572,572,572,572,572,584,572,584,572,572,572,572,572,572, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, +572,572,572,572,572,572,572,572,572,572,572,572, 20, 20, 20, 20, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,572, /* block 96 */ -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,576,576,576,576,576,576,576,576, -576,576,576,576,576,576,576,576,563,563,563,563,563,563,563,563, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,563,563,563,563,563, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,585,585,585,585,585,585,585,585, +585,585,585,585,585,585,585,585,572,572,572,572,572,572,572,572, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,572,572,572,572,572, /* block 97 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, @@ -2641,325 +2695,325 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, -563,563,563,563,563,563,563,563,563,563,563,563,563,563,563, 20, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, +572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, 20, /* block 98 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, /* block 99 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,119,119,119,119,119,119,119,119,119,119, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, /* block 100 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,120,120,120, /* block 101 */ -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,579,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,588,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, /* block 102 */ -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, -578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, +587,587,587,587,587,587,587,587,587,587,587,587,587,587,587,587, /* block 103 */ -578,578,578,578,578,578,578,578,578,578,578,578,578,119,119,119, -580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, -580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, -580,580,580,580,580,580,580,580,580,580,580,580,580,580,580,580, -580,580,580,580,580,580,580,119,119,119,119,119,119,119,119,119, -581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, -581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, -581,581,581,581,581,581,581,581,582,582,582,582,582,582,583,583, +587,587,587,587,587,587,587,587,587,587,587,587,587,120,120,120, +589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,589, +589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,589, +589,589,589,589,589,589,589,589,589,589,589,589,589,589,589,589, +589,589,589,589,589,589,589,120,120,120,120,120,120,120,120,120, +590,590,590,590,590,590,590,590,590,590,590,590,590,590,590,590, +590,590,590,590,590,590,590,590,590,590,590,590,590,590,590,590, +590,590,590,590,590,590,590,590,591,591,591,591,591,591,592,592, /* block 104 */ -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, /* block 105 */ -584,584,584,584,584,584,584,584,584,584,584,584,585,586,586,586, -584,584,584,584,584,584,584,584,584,584,584,584,584,584,584,584, -587,587,587,587,587,587,587,587,587,587,584,584,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -191,192,191,192,191,192,191,192,191,192,588,589,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,191,192,590,197, -199,199,199,591,543,543,543,543,543,543,543,543,543,543,591,472, +593,593,593,593,593,593,593,593,593,593,593,593,594,595,595,595, +593,593,593,593,593,593,593,593,593,593,593,593,593,593,593,593, +596,596,596,596,596,596,596,596,596,596,593,593,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +192,193,192,193,192,193,192,193,192,193,597,598,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,599,198, +200,200,200,600,552,552,552,552,552,552,552,552,552,552,600,478, /* block 106 */ -191,192,191,192,191,192,191,192,191,192,191,192,191,192,191,192, -191,192,191,192,191,192,191,192,191,192,191,192,472,472,543,543, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,593,593,593,593,593,593,593,593,593,593, -594,594,595,595,595,595,595,595,119,119,119,119,119,119,119,119, +192,193,192,193,192,193,192,193,192,193,192,193,192,193,192,193, +192,193,192,193,192,193,192,193,192,193,192,193,478,478,552,552, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,602,602,602,602,602,602,602,602,602,602, +603,603,604,604,604,604,604,604,120,120,120,120,120,120,120,120, /* block 107 */ - 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, - 15, 15, 15, 15, 15, 15, 15,110,110,110,110,110,110,110,110,110, +605,605,605,605,605,605,605,605, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15,111,111,111,111,111,111,111,111,111, 15, 15, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 35, 35, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, -109, 35, 35, 35, 35, 35, 35, 35, 35, 32, 33, 32, 33,596, 32, 33, +110, 35, 35, 35, 35, 35, 35, 35, 35, 32, 33, 32, 33,606, 32, 33, /* block 108 */ - 32, 33, 32, 33, 32, 33, 32, 33,110, 15, 15, 32, 33,597, 35, 22, - 32, 33, 32, 33, 35, 35, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, - 32, 33, 32, 33, 32, 33, 32, 33, 32, 33,598,599,600,601,598, 35, -602,603,604,605, 32, 33, 32, 33, 32, 33,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119, 22,109,109, 35, 22, 22, 22, 22, 22, + 32, 33, 32, 33, 32, 33, 32, 33,111, 15, 15, 32, 33,607, 35, 22, + 32, 33, 32, 33,608, 35, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, + 32, 33, 32, 33, 32, 33, 32, 33, 32, 33,609,610,611,612,609, 35, +613,614,615,616, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, 32, 33, +120,120, 32, 33,617,618,619, 32, 33, 32, 33,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120, 32, 33, 22,110,110, 35, 22, 22, 22, 22, 22, /* block 109 */ -606,606,607,606,606,606,607,606,606,606,606,607,606,606,606,606, -606,606,606,606,606,606,606,606,606,606,606,606,606,606,606,606, -606,606,606,608,608,607,607,608,609,609,609,609,119,119,119,119, -610,610,610,611,611,611,612,612,613,612,119,119,119,119,119,119, -614,614,614,614,614,614,614,614,614,614,614,614,614,614,614,614, -614,614,614,614,614,614,614,614,614,614,614,614,614,614,614,614, -614,614,614,614,614,614,614,614,614,614,614,614,614,614,614,614, -614,614,614,614,615,615,615,615,119,119,119,119,119,119,119,119, +620,620,621,620,620,620,621,620,620,620,620,621,620,620,620,620, +620,620,620,620,620,620,620,620,620,620,620,620,620,620,620,620, +620,620,620,622,622,621,621,622,623,623,623,623,621,120,120,120, +624,624,624,625,625,625,626,626,627,626,120,120,120,120,120,120, +628,628,628,628,628,628,628,628,628,628,628,628,628,628,628,628, +628,628,628,628,628,628,628,628,628,628,628,628,628,628,628,628, +628,628,628,628,628,628,628,628,628,628,628,628,628,628,628,628, +628,628,628,628,629,629,629,629,120,120,120,120,120,120,120,120, /* block 110 */ -616,616,617,617,617,617,617,617,617,617,617,617,617,617,617,617, -617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617, -617,617,617,617,617,617,617,617,617,617,617,617,617,617,617,617, -617,617,617,617,616,616,616,616,616,616,616,616,616,616,616,616, -616,616,616,616,618,618,119,119,119,119,119,119,119,119,619,619, -620,620,620,620,620,620,620,620,620,620,119,119,119,119,119,119, +630,630,631,631,631,631,631,631,631,631,631,631,631,631,631,631, +631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,631, +631,631,631,631,631,631,631,631,631,631,631,631,631,631,631,631, +631,631,631,631,630,630,630,630,630,630,630,630,630,630,630,630, +630,630,630,630,632,632,120,120,120,120,120,120,120,120,633,633, +634,634,634,634,634,634,634,634,634,634,120,120,120,120,120,120, 251,251,251,251,251,251,251,251,251,251,251,251,251,251,251,251, -251,621,253,622,253,253,253,253,259,259,259,253,259,253,253,251, +251,635,253,636,253,253,253,253,259,259,259,253,259,253,253,251, /* block 111 */ -623,623,623,623,623,623,623,623,623,623,624,624,624,624,624,624, -624,624,624,624,624,624,624,624,624,624,624,624,624,624,624,624, -624,624,624,624,624,624,625,625,625,625,625,625,625,625,626,627, -628,628,628,628,628,628,628,628,628,628,628,628,628,628,628,628, -628,628,628,628,628,628,628,629,629,629,629,629,629,629,629,629, -629,629,630,630,119,119,119,119,119,119,119,119,119,119,119,631, -355,355,355,355,355,355,355,355,355,355,355,355,355,355,355,355, -355,355,355,355,355,355,355,355,355,355,355,355,355,119,119,119, +637,637,637,637,637,637,637,637,637,637,638,638,638,638,638,638, +638,638,638,638,638,638,638,638,638,638,638,638,638,638,638,638, +638,638,638,638,638,638,639,639,639,639,639,639,639,639,640,641, +642,642,642,642,642,642,642,642,642,642,642,642,642,642,642,642, +642,642,642,642,642,642,642,643,643,643,643,643,643,643,643,643, +643,643,644,644,120,120,120,120,120,120,120,120,120,120,120,645, +356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, +356,356,356,356,356,356,356,356,356,356,356,356,356,120,120,120, /* block 112 */ -632,632,632,633,634,634,634,634,634,634,634,634,634,634,634,634, -634,634,634,634,634,634,634,634,634,634,634,634,634,634,634,634, -634,634,634,634,634,634,634,634,634,634,634,634,634,634,634,634, -634,634,634,632,633,633,632,632,632,632,633,633,632,633,633,633, -633,635,635,635,635,635,635,635,635,635,635,635,635,635,119,636, -637,637,637,637,637,637,637,637,637,637,119,119,119,119,635,635, -343,343,343,343,343,345,638,343,343,343,343,343,343,343,343,343, -349,349,349,349,349,349,349,349,349,349,343,343,343,343,343,119, +646,646,646,647,648,648,648,648,648,648,648,648,648,648,648,648, +648,648,648,648,648,648,648,648,648,648,648,648,648,648,648,648, +648,648,648,648,648,648,648,648,648,648,648,648,648,648,648,648, +648,648,648,646,647,647,646,646,646,646,647,647,646,646,647,647, +647,649,649,649,649,649,649,649,649,649,649,649,649,649,120,650, +651,651,651,651,651,651,651,651,651,651,120,120,120,120,649,649, +344,344,344,344,344,346,652,344,344,344,344,344,344,344,344,344, +350,350,350,350,350,350,350,350,350,350,344,344,344,344,344,120, /* block 113 */ -639,639,639,639,639,639,639,639,639,639,639,639,639,639,639,639, -639,639,639,639,639,639,639,639,639,639,639,639,639,639,639,639, -639,639,639,639,639,639,639,639,639,640,640,640,640,640,640,641, -641,640,640,641,641,640,640,119,119,119,119,119,119,119,119,119, -639,639,639,640,639,639,639,639,639,639,639,639,640,641,119,119, -642,642,642,642,642,642,642,642,642,642,119,119,643,643,643,643, -343,343,343,343,343,343,343,343,343,343,343,343,343,343,343,343, -638,343,343,343,343,343,343,350,350,350,343,344,345,344,343,343, +653,653,653,653,653,653,653,653,653,653,653,653,653,653,653,653, +653,653,653,653,653,653,653,653,653,653,653,653,653,653,653,653, +653,653,653,653,653,653,653,653,653,654,654,654,654,654,654,655, +655,654,654,655,655,654,654,120,120,120,120,120,120,120,120,120, +653,653,653,654,653,653,653,653,653,653,653,653,654,655,120,120, +656,656,656,656,656,656,656,656,656,656,120,120,657,657,657,657, +344,344,344,344,344,344,344,344,344,344,344,344,344,344,344,344, +652,344,344,344,344,344,344,351,351,351,344,345,346,345,344,344, /* block 114 */ -644,644,644,644,644,644,644,644,644,644,644,644,644,644,644,644, -644,644,644,644,644,644,644,644,644,644,644,644,644,644,644,644, -644,644,644,644,644,644,644,644,644,644,644,644,644,644,644,644, -645,644,645,645,645,644,644,645,645,644,644,644,644,644,645,645, -644,645,644,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,644,644,646,647,647, -648,648,648,648,648,648,648,648,648,648,648,649,650,650,649,649, -651,651,648,652,652,649,650,119,119,119,119,119,119,119,119,119, +658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, +658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, +658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, +659,658,659,659,659,658,658,659,659,658,658,658,658,658,659,659, +658,659,658,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,658,658,660,661,661, +662,662,662,662,662,662,662,662,662,662,662,663,664,664,663,663, +665,665,662,666,666,663,664,120,120,120,120,120,120,120,120,120, /* block 115 */ -119,358,358,358,358,358,358,119,119,358,358,358,358,358,358,119, -119,358,358,358,358,358,358,119,119,119,119,119,119,119,119,119, -358,358,358,358,358,358,358,119,358,358,358,358,358,358,358,119, +120,359,359,359,359,359,359,120,120,359,359,359,359,359,359,120, +120,359,359,359,359,359,359,120,120,120,120,120,120,120,120,120, +359,359,359,359,359,359,359,120,359,359,359,359,359,359,359,120, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, - 35, 35, 35,653, 35, 35, 35, 35, 35, 35, 35, 15,109,109,109,109, - 35, 35, 35, 35, 35,127,119,119,119,119,119,119,119,119,119,119, -654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654, + 35, 35, 35,667, 35, 35, 35, 35, 35, 35, 35, 15,110,110,110,110, + 35, 35, 35, 35, 35,128, 35, 35, 35,110, 15, 15,120,120,120,120, +668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668, /* block 116 */ -654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654, -654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654, -654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654, -654,654,654,654,654,654,654,654,654,654,654,654,654,654,654,654, -648,648,648,648,648,648,648,648,648,648,648,648,648,648,648,648, -648,648,648,648,648,648,648,648,648,648,648,648,648,648,648,648, -648,648,648,649,649,650,649,649,650,649,649,651,649,650,119,119, -655,655,655,655,655,655,655,655,655,655,119,119,119,119,119,119, +668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668, +668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668, +668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668, +668,668,668,668,668,668,668,668,668,668,668,668,668,668,668,668, +662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,662, +662,662,662,662,662,662,662,662,662,662,662,662,662,662,662,662, +662,662,662,663,663,664,663,663,664,663,663,665,663,664,120,120, +669,669,669,669,669,669,669,669,669,669,120,120,120,120,120,120, /* block 117 */ -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, /* block 118 */ -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, /* block 119 */ -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, /* block 120 */ -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, /* block 121 */ -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, /* block 122 */ -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, /* block 123 */ -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -656,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,656,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,656,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +670,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,670,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,670,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, /* block 124 */ -657,657,657,657,657,657,657,657,656,657,657,657,657,657,657,657, -657,657,657,657,657,657,657,657,657,657,657,657,657,657,657,657, -657,657,657,657,119,119,119,119,119,119,119,119,119,119,119,119, -356,356,356,356,356,356,356,356,356,356,356,356,356,356,356,356, -356,356,356,356,356,356,356,119,119,119,119,357,357,357,357,357, -357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, +671,671,671,671,671,671,671,671,670,671,671,671,671,671,671,671, +671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, +671,671,671,671,120,120,120,120,120,120,120,120,120,120,120,120, 357,357,357,357,357,357,357,357,357,357,357,357,357,357,357,357, -357,357,357,357,357,357,357,357,357,357,357,357,119,119,119,119, +357,357,357,357,357,357,357,120,120,120,120,358,358,358,358,358, +358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, +358,358,358,358,358,358,358,358,358,358,358,358,358,358,358,358, +358,358,358,358,358,358,358,358,358,358,358,358,120,120,120,120, /* block 125 */ -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, -658,658,658,658,658,658,658,658,658,658,658,658,658,658,658,658, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, +672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, /* block 126 */ -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, /* block 127 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,119,119, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,120,120, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, /* block 128 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 129 */ - 35, 35, 35, 35, 35, 35, 35,119,119,119,119,119,119,119,119,119, -119,119,119,205,205,205,205,205,119,119,119,119,119,214,211,214, -214,214,214,214,214,214,214,214,214,660,214,214,214,214,214,214, -214,214,214,214,214,214,214,119,214,214,214,214,214,119,214,119, -214,214,119,214,214,119,214,214,214,214,214,214,214,214,214,214, + 35, 35, 35, 35, 35, 35, 35,120,120,120,120,120,120,120,120,120, +120,120,120,206,206,206,206,206,120,120,120,120,120,214,211,214, +214,214,214,214,214,214,214,214,214,674,214,214,214,214,214,214, +214,214,214,214,214,214,214,120,214,214,214,214,214,120,214,120, +214,214,120,214,214,120,214,214,214,214,214,214,214,214,214,214, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, @@ -2968,9 +3022,9 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -224,224,661,661,661,661,661,661,661,661,661,661,661,661,661,661, -661,661,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,224,224,224,224,224,224,224,224,224,224,224,224,224, +224,224,675,675,675,675,675,675,675,675,675,675,675,675,675,675, +675,675,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, @@ -2989,30 +3043,30 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224, 8, 7, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, /* block 133 */ 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -119,119,224,224,224,224,224,224,224,224,224,224,224,224,224,224, +120,120,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -224,224,224,224,224,224,224,224,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -224,224,662,224,224,224,224,224,224,224,224,224,219,663,119,119, +224,224,224,224,224,224,224,224,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +224,224,676,224,224,224,224,224,224,224,224,224,219,677,120,120, /* block 134 */ -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, - 5, 5, 5, 5, 5, 5, 5, 7, 8, 5,119,119,119,119,119,119, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,543,543, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, + 5, 5, 5, 5, 5, 5, 5, 7, 8, 5,120,120,120,120,120,120, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,552,552, 5, 10, 10, 16, 16, 7, 8, 7, 8, 7, 8, 7, 8, 7, 8, 7, - 8, 7, 8, 7, 8,547,547, 7, 8, 5, 5, 5, 5, 16, 16, 16, - 5, 5, 5,119, 5, 5, 5, 5, 10, 7, 8, 7, 8, 7, 8, 5, - 5, 5, 9, 10, 9, 9, 9,119, 5, 6, 5, 5,119,119,119,119, -224,224,224,224,224,119,224,224,224,224,224,224,224,224,224,224, + 8, 7, 8, 7, 8,556,556, 7, 8, 5, 5, 5, 5, 16, 16, 16, + 5, 5, 5,120, 5, 5, 5, 5, 10, 7, 8, 7, 8, 7, 8, 5, + 5, 5, 9, 10, 9, 9, 9,120, 5, 6, 5, 5,120,120,120,120, +224,224,224,224,224,120,224,224,224,224,224,224,224,224,224,224, /* block 135 */ 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, @@ -3022,779 +3076,849 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -224,224,224,224,224,224,224,224,224,224,224,224,224,119,119, 24, +224,224,224,224,224,224,224,224,224,224,224,224,224,120,120, 24, /* block 136 */ -119, 5, 5, 5, 6, 5, 5, 5, 7, 8, 5, 9, 5, 10, 5, 5, +120, 5, 5, 5, 6, 5, 5, 5, 7, 8, 5, 9, 5, 10, 5, 5, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 5, 5, 9, 9, 9, 5, 5, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 7, 5, 8, 15, 16, 15, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 7, 9, 8, 9, 7, - 8,546,551,552,546,546,569,569,569,569,569,569,569,569,569,569, -560,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, + 8,555,560,561,555,555,578,578,578,578,578,578,578,578,578,578, +569,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, /* block 137 */ -569,569,569,569,569,569,569,569,569,569,569,569,569,569,569,569, -569,569,569,569,569,569,569,569,569,569,569,569,569,569,664,664, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,572, -572,572,572,572,572,572,572,572,572,572,572,572,572,572,572,119, -119,119,572,572,572,572,572,572,119,119,572,572,572,572,572,572, -119,119,572,572,572,572,572,572,119,119,572,572,572,119,119,119, - 6, 6, 9, 15, 20, 6, 6,119, 20, 9, 9, 9, 9, 20, 20,119, -502,502,502,502,502,502,502,502,502, 24, 24, 24, 20, 20,119,119, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,578,578, +578,578,578,578,578,578,578,578,578,578,578,578,578,578,678,678, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,581, +581,581,581,581,581,581,581,581,581,581,581,581,581,581,581,120, +120,120,581,581,581,581,581,581,120,120,581,581,581,581,581,581, +120,120,581,581,581,581,581,581,120,120,581,581,581,120,120,120, + 6, 6, 9, 15, 20, 6, 6,120, 20, 9, 9, 9, 9, 20, 20,120, +511,511,511,511,511,511,511,511,511, 24, 24, 24, 20, 20,120,120, /* block 138 */ -665,665,665,665,665,665,665,665,665,665,665,665,119,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,119,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,119,665,665,119,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,119,119, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +679,679,679,679,679,679,679,679,679,679,679,679,120,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,120,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,120,679,679,120,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,120,120, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 139 */ -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,665,665,665,665,665, -665,665,665,665,665,665,665,665,665,665,665,119,119,119,119,119, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,679,679,679,679,679, +679,679,679,679,679,679,679,679,679,679,679,120,120,120,120,120, /* block 140 */ -666,666,666,119,119,119,119,667,667,667,667,667,667,667,667,667, -667,667,667,667,667,667,667,667,667,667,667,667,667,667,667,667, -667,667,667,667,667,667,667,667,667,667,667,667,667,667,667,667, -667,667,667,667,119,119,119,668,668,668,668,668,668,668,668,668, -669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669, -669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669, -669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669, -669,669,669,669,669,670,670,670,670,671,671,671,671,671,671,671, +680,680,680,120,120,120,120,681,681,681,681,681,681,681,681,681, +681,681,681,681,681,681,681,681,681,681,681,681,681,681,681,681, +681,681,681,681,681,681,681,681,681,681,681,681,681,681,681,681, +681,681,681,681,120,120,120,682,682,682,682,682,682,682,682,682, +683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,683, +683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,683, +683,683,683,683,683,683,683,683,683,683,683,683,683,683,683,683, +683,683,683,683,683,684,684,684,684,685,685,685,685,685,685,685, /* block 141 */ -671,671,671,671,671,671,671,671,671,671,670,670,671,671,671,119, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,119,119,119,119, -671,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +685,685,685,685,685,685,685,685,685,685,684,684,685,685,685,120, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,120,120,120, +685,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,112,119,119, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,113,120,120, /* block 142 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 143 */ -672,672,672,672,672,672,672,672,672,672,672,672,672,672,672,672, -672,672,672,672,672,672,672,672,672,672,672,672,672,119,119,119, -673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, -673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, -673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, -673,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -674,675,675,675,675,675,675,675,675,675,675,675,675,675,675,675, -675,675,675,675,675,675,675,675,675,675,675,675,119,119,119,119, +686,686,686,686,686,686,686,686,686,686,686,686,686,686,686,686, +686,686,686,686,686,686,686,686,686,686,686,686,686,120,120,120, +687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687, +687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687, +687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687, +687,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +688,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689, +689,689,689,689,689,689,689,689,689,689,689,689,120,120,120,120, /* block 144 */ -676,676,676,676,676,676,676,676,676,676,676,676,676,676,676,676, -676,676,676,676,676,676,676,676,676,676,676,676,676,676,676,676, -677,677,677,677,119,119,119,119,119,119,119,119,119,676,676,676, -678,678,678,678,678,678,678,678,678,678,678,678,678,678,678,678, -678,679,678,678,678,678,678,678,678,678,679,119,119,119,119,119, -680,680,680,680,680,680,680,680,680,680,680,680,680,680,680,680, -680,680,680,680,680,680,680,680,680,680,680,680,680,680,680,680, -680,680,680,680,680,680,681,681,681,681,681,119,119,119,119,119, +690,690,690,690,690,690,690,690,690,690,690,690,690,690,690,690, +690,690,690,690,690,690,690,690,690,690,690,690,690,690,690,690, +691,691,691,691,120,120,120,120,120,120,120,120,120,690,690,690, +692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,692, +692,693,692,692,692,692,692,692,692,692,693,120,120,120,120,120, +694,694,694,694,694,694,694,694,694,694,694,694,694,694,694,694, +694,694,694,694,694,694,694,694,694,694,694,694,694,694,694,694, +694,694,694,694,694,694,695,695,695,695,695,120,120,120,120,120, /* block 145 */ -682,682,682,682,682,682,682,682,682,682,682,682,682,682,682,682, -682,682,682,682,682,682,682,682,682,682,682,682,682,682,119,683, -684,684,684,684,684,684,684,684,684,684,684,684,684,684,684,684, -684,684,684,684,684,684,684,684,684,684,684,684,684,684,684,684, -684,684,684,684,119,119,119,119,684,684,684,684,684,684,684,684, -685,686,686,686,686,686,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +696,696,696,696,696,696,696,696,696,696,696,696,696,696,696,696, +696,696,696,696,696,696,696,696,696,696,696,696,696,696,120,697, +698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698, +698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698, +698,698,698,698,120,120,120,120,698,698,698,698,698,698,698,698, +699,700,700,700,700,700,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 146 */ -687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687, -687,687,687,687,687,687,687,687,687,687,687,687,687,687,687,687, -687,687,687,687,687,687,687,687,688,688,688,688,688,688,688,688, -688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688, -688,688,688,688,688,688,688,688,688,688,688,688,688,688,688,688, -689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689, -689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689, -689,689,689,689,689,689,689,689,689,689,689,689,689,689,689,689, +701,701,701,701,701,701,701,701,701,701,701,701,701,701,701,701, +701,701,701,701,701,701,701,701,701,701,701,701,701,701,701,701, +701,701,701,701,701,701,701,701,702,702,702,702,702,702,702,702, +702,702,702,702,702,702,702,702,702,702,702,702,702,702,702,702, +702,702,702,702,702,702,702,702,702,702,702,702,702,702,702,702, +703,703,703,703,703,703,703,703,703,703,703,703,703,703,703,703, +703,703,703,703,703,703,703,703,703,703,703,703,703,703,703,703, +703,703,703,703,703,703,703,703,703,703,703,703,703,703,703,703, /* block 147 */ -690,690,690,690,690,690,690,690,690,690,690,690,690,690,690,690, -690,690,690,690,690,690,690,690,690,690,690,690,690,690,119,119, -691,691,691,691,691,691,691,691,691,691,119,119,119,119,119,119, -692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,692, -692,692,692,692,692,692,692,692,692,692,692,692,692,692,692,692, -692,692,692,692,119,119,119,119,693,693,693,693,693,693,693,693, -693,693,693,693,693,693,693,693,693,693,693,693,693,693,693,693, -693,693,693,693,693,693,693,693,693,693,693,693,119,119,119,119, +704,704,704,704,704,704,704,704,704,704,704,704,704,704,704,704, +704,704,704,704,704,704,704,704,704,704,704,704,704,704,120,120, +705,705,705,705,705,705,705,705,705,705,120,120,120,120,120,120, +706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,706, +706,706,706,706,706,706,706,706,706,706,706,706,706,706,706,706, +706,706,706,706,120,120,120,120,707,707,707,707,707,707,707,707, +707,707,707,707,707,707,707,707,707,707,707,707,707,707,707,707, +707,707,707,707,707,707,707,707,707,707,707,707,120,120,120,120, /* block 148 */ -694,694,694,694,694,694,694,694,694,694,694,694,694,694,694,694, -694,694,694,694,694,694,694,694,694,694,694,694,694,694,694,694, -694,694,694,694,694,694,694,694,119,119,119,119,119,119,119,119, -695,695,695,695,695,695,695,695,695,695,695,695,695,695,695,695, -695,695,695,695,695,695,695,695,695,695,695,695,695,695,695,695, -695,695,695,695,695,695,695,695,695,695,695,695,695,695,695,695, -695,695,695,695,119,119,119,119,119,119,119,119,119,119,119,696, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +708,708,708,708,708,708,708,708,708,708,708,708,708,708,708,708, +708,708,708,708,708,708,708,708,708,708,708,708,708,708,708,708, +708,708,708,708,708,708,708,708,120,120,120,120,120,120,120,120, +709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709, +709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709, +709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709, +709,709,709,709,120,120,120,120,120,120,120,120,120,120,120,710, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 149 */ -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, /* block 150 */ -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,697,119,119,119,119,119,119,119,119,119, -697,697,697,697,697,697,697,697,697,697,697,697,697,697,697,697, -697,697,697,697,697,697,119,119,119,119,119,119,119,119,119,119, -697,697,697,697,697,697,697,697,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,711,120,120,120,120,120,120,120,120,120, +711,711,711,711,711,711,711,711,711,711,711,711,711,711,711,711, +711,711,711,711,711,711,120,120,120,120,120,120,120,120,120,120, +711,711,711,711,711,711,711,711,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 151 */ -698,698,698,698,698,698,119,119,698,119,698,698,698,698,698,698, -698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698, -698,698,698,698,698,698,698,698,698,698,698,698,698,698,698,698, -698,698,698,698,698,698,119,698,698,119,119,119,698,119,119,698, -699,699,699,699,699,699,699,699,699,699,699,699,699,699,699,699, -699,699,699,699,699,699,119,700,701,701,701,701,701,701,701,701, -702,702,702,702,702,702,702,702,702,702,702,702,702,702,702,702, -702,702,702,702,702,702,702,703,703,704,704,704,704,704,704,704, +712,712,712,712,712,712,120,120,712,120,712,712,712,712,712,712, +712,712,712,712,712,712,712,712,712,712,712,712,712,712,712,712, +712,712,712,712,712,712,712,712,712,712,712,712,712,712,712,712, +712,712,712,712,712,712,120,712,712,120,120,120,712,120,120,712, +713,713,713,713,713,713,713,713,713,713,713,713,713,713,713,713, +713,713,713,713,713,713,120,714,715,715,715,715,715,715,715,715, +716,716,716,716,716,716,716,716,716,716,716,716,716,716,716,716, +716,716,716,716,716,716,716,717,717,718,718,718,718,718,718,718, /* block 152 */ -705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,705, -705,705,705,705,705,705,705,705,705,705,705,705,705,705,705,119, -119,119,119,119,119,119,119,706,706,706,706,706,706,706,706,706, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -707,707,707,707,707,707,707,707,707,707,707,707,707,707,707,707, -707,707,707,119,707,707,119,119,119,119,119,708,708,708,708,708, +719,719,719,719,719,719,719,719,719,719,719,719,719,719,719,719, +719,719,719,719,719,719,719,719,719,719,719,719,719,719,719,120, +120,120,120,120,120,120,120,720,720,720,720,720,720,720,720,720, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +721,721,721,721,721,721,721,721,721,721,721,721,721,721,721,721, +721,721,721,120,721,721,120,120,120,120,120,722,722,722,722,722, /* block 153 */ -709,709,709,709,709,709,709,709,709,709,709,709,709,709,709,709, -709,709,709,709,709,709,710,710,710,710,710,710,119,119,119,711, -712,712,712,712,712,712,712,712,712,712,712,712,712,712,712,712, -712,712,712,712,712,712,712,712,712,712,119,119,119,119,119,713, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +723,723,723,723,723,723,723,723,723,723,723,723,723,723,723,723, +723,723,723,723,723,723,724,724,724,724,724,724,120,120,120,725, +726,726,726,726,726,726,726,726,726,726,726,726,726,726,726,726, +726,726,726,726,726,726,726,726,726,726,120,120,120,120,120,727, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 154 */ -714,714,714,714,714,714,714,714,714,714,714,714,714,714,714,714, -714,714,714,714,714,714,714,714,714,714,714,714,714,714,714,714, -715,715,715,715,715,715,715,715,715,715,715,715,715,715,715,715, -715,715,715,715,715,715,715,715,119,119,119,119,716,716,715,715, -716,716,716,716,716,716,716,716,716,716,716,716,716,716,716,716, -119,119,716,716,716,716,716,716,716,716,716,716,716,716,716,716, -716,716,716,716,716,716,716,716,716,716,716,716,716,716,716,716, -716,716,716,716,716,716,716,716,716,716,716,716,716,716,716,716, +728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728, +728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728, +729,729,729,729,729,729,729,729,729,729,729,729,729,729,729,729, +729,729,729,729,729,729,729,729,120,120,120,120,730,730,729,729, +730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730, +120,120,730,730,730,730,730,730,730,730,730,730,730,730,730,730, +730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730, +730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730, /* block 155 */ -717,718,718,718,119,718,718,119,119,119,119,119,718,718,718,718, -717,717,717,717,119,717,717,717,119,717,717,717,717,717,717,717, -717,717,717,717,717,717,717,717,717,717,717,717,717,717,717,717, -717,717,717,717,717,717,119,119,718,718,718,119,119,119,119,718, -719,719,719,719,719,719,719,719,719,119,119,119,119,119,119,119, -720,720,720,720,720,720,720,720,720,119,119,119,119,119,119,119, -721,721,721,721,721,721,721,721,721,721,721,721,721,721,721,721, -721,721,721,721,721,721,721,721,721,721,721,721,721,722,722,723, +731,732,732,732,120,732,732,120,120,120,120,120,732,732,732,732, +731,731,731,731,120,731,731,731,120,731,731,731,731,731,731,731, +731,731,731,731,731,731,731,731,731,731,731,731,731,731,731,731, +731,731,731,731,731,731,120,120,732,732,732,120,120,120,120,732, +733,733,733,733,733,733,733,733,733,120,120,120,120,120,120,120, +734,734,734,734,734,734,734,734,734,120,120,120,120,120,120,120, +735,735,735,735,735,735,735,735,735,735,735,735,735,735,735,735, +735,735,735,735,735,735,735,735,735,735,735,735,735,736,736,737, /* block 156 */ -724,724,724,724,724,724,724,724,724,724,724,724,724,724,724,724, -724,724,724,724,724,724,724,724,724,724,724,724,724,725,725,725, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -726,726,726,726,726,726,726,726,727,726,726,726,726,726,726,726, -726,726,726,726,726,726,726,726,726,726,726,726,726,726,726,726, -726,726,726,726,726,728,728,119,119,119,119,729,729,729,729,729, -730,730,730,730,730,730,730,119,119,119,119,119,119,119,119,119, +738,738,738,738,738,738,738,738,738,738,738,738,738,738,738,738, +738,738,738,738,738,738,738,738,738,738,738,738,738,739,739,739, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +740,740,740,740,740,740,740,740,741,740,740,740,740,740,740,740, +740,740,740,740,740,740,740,740,740,740,740,740,740,740,740,740, +740,740,740,740,740,742,742,120,120,120,120,743,743,743,743,743, +744,744,744,744,744,744,744,120,120,120,120,120,120,120,120,120, /* block 157 */ -731,731,731,731,731,731,731,731,731,731,731,731,731,731,731,731, -731,731,731,731,731,731,731,731,731,731,731,731,731,731,731,731, -731,731,731,731,731,731,731,731,731,731,731,731,731,731,731,731, -731,731,731,731,731,731,119,119,119,732,732,732,732,732,732,732, -733,733,733,733,733,733,733,733,733,733,733,733,733,733,733,733, -733,733,733,733,733,733,119,119,734,734,734,734,734,734,734,734, -735,735,735,735,735,735,735,735,735,735,735,735,735,735,735,735, -735,735,735,119,119,119,119,119,736,736,736,736,736,736,736,736, +745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745, +745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745, +745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745, +745,745,745,745,745,745,120,120,120,746,746,746,746,746,746,746, +747,747,747,747,747,747,747,747,747,747,747,747,747,747,747,747, +747,747,747,747,747,747,120,120,748,748,748,748,748,748,748,748, +749,749,749,749,749,749,749,749,749,749,749,749,749,749,749,749, +749,749,749,120,120,120,120,120,750,750,750,750,750,750,750,750, /* block 158 */ -737,737,737,737,737,737,737,737,737,737,737,737,737,737,737,737, -737,737,119,119,119,119,119,119,119,738,738,738,738,119,119,119, -119,119,119,119,119,119,119,119,119,739,739,739,739,739,739,739, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +751,751,751,751,751,751,751,751,751,751,751,751,751,751,751,751, +751,751,120,120,120,120,120,120,120,752,752,752,752,120,120,120, +120,120,120,120,120,120,120,120,120,753,753,753,753,753,753,753, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 159 */ -740,740,740,740,740,740,740,740,740,740,740,740,740,740,740,740, -740,740,740,740,740,740,740,740,740,740,740,740,740,740,740,740, -740,740,740,740,740,740,740,740,740,740,740,740,740,740,740,740, -740,740,740,740,740,740,740,740,740,740,740,740,740,740,740,740, -740,740,740,740,740,740,740,740,740,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754, +754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754, +754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754, +754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754, +754,754,754,754,754,754,754,754,754,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 160 */ -741,741,741,741,741,741,741,741,741,741,741,741,741,741,741,741, -741,741,741,741,741,741,741,741,741,741,741,741,741,741,741,741, -741,741,741,741,741,741,741,741,741,741,741,741,741,741,741,741, -741,741,741,119,119,119,119,119,119,119,119,119,119,119,119,119, -742,742,742,742,742,742,742,742,742,742,742,742,742,742,742,742, -742,742,742,742,742,742,742,742,742,742,742,742,742,742,742,742, -742,742,742,742,742,742,742,742,742,742,742,742,742,742,742,742, -742,742,742,119,119,119,119,119,119,119,743,743,743,743,743,743, +755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755, +755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755, +755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755, +755,755,755,120,120,120,120,120,120,120,120,120,120,120,120,120, +756,756,756,756,756,756,756,756,756,756,756,756,756,756,756,756, +756,756,756,756,756,756,756,756,756,756,756,756,756,756,756,756, +756,756,756,756,756,756,756,756,756,756,756,756,756,756,756,756, +756,756,756,120,120,120,120,120,120,120,757,757,757,757,757,757, /* block 161 */ -744,744,744,744,744,744,744,744,744,744,744,744,744,744,744,744, -744,744,744,744,744,744,744,744,744,744,744,744,744,744,744,744, -744,744,744,744,745,745,745,745,119,119,119,119,119,119,119,119, -746,746,746,746,746,746,746,746,746,746,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +758,758,758,758,758,758,758,758,758,758,758,758,758,758,758,758, +758,758,758,758,758,758,758,758,758,758,758,758,758,758,758,758, +758,758,758,758,759,759,759,759,120,120,120,120,120,120,120,120, +760,760,760,760,760,760,760,760,760,760,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 162 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -747,747,747,747,747,747,747,747,747,747,747,747,747,747,747,747, -747,747,747,747,747,747,747,747,747,747,747,747,747,747,747,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +761,761,761,761,761,761,761,761,761,761,761,761,761,761,761,761, +761,761,761,761,761,761,761,761,761,761,761,761,761,761,761,120, /* block 163 */ -748,748,748,748,748,748,748,748,748,748,748,748,748,748,748,748, -748,748,748,748,748,748,748,748,748,748,748,748,748,749,749,749, -749,749,749,749,749,749,749,748,119,119,119,119,119,119,119,119, -750,750,750,750,750,750,750,750,750,750,750,750,750,750,750,750, -750,750,750,750,750,750,751,751,751,751,751,751,751,751,751,751, -751,752,752,752,752,753,753,753,753,753,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +762,762,762,762,762,762,762,762,762,762,762,762,762,762,762,762, +762,762,762,762,762,762,762,762,762,762,762,762,762,762,762,762, +762,762,762,762,762,762,762,762,762,762,120,763,763,764,120,120, +762,762,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 164 */ -754,755,754,756,756,756,756,756,756,756,756,756,756,756,756,756, -756,756,756,756,756,756,756,756,756,756,756,756,756,756,756,756, -756,756,756,756,756,756,756,756,756,756,756,756,756,756,756,756, -756,756,756,756,756,756,756,756,755,755,755,755,755,755,755,755, -755,755,755,755,755,755,755,757,757,757,757,757,757,757,119,119, -119,119,758,758,758,758,758,758,758,758,758,758,758,758,758,758, -758,758,758,758,758,758,759,759,759,759,759,759,759,759,759,759, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,755, +765,765,765,765,765,765,765,765,765,765,765,765,765,765,765,765, +765,765,765,765,765,765,765,765,765,765,765,765,765,766,766,766, +766,766,766,766,766,766,766,765,120,120,120,120,120,120,120,120, +767,767,767,767,767,767,767,767,767,767,767,767,767,767,767,767, +767,767,767,767,767,767,768,768,768,768,768,768,768,768,768,768, +768,769,769,769,769,770,770,770,770,770,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 165 */ -760,760,761,762,762,762,762,762,762,762,762,762,762,762,762,762, -762,762,762,762,762,762,762,762,762,762,762,762,762,762,762,762, -762,762,762,762,762,762,762,762,762,762,762,762,762,762,762,762, -761,761,761,760,760,760,760,761,761,760,760,763,763,764,763,763, -763,763,119,119,119,119,119,119,119,119,119,119,119,764,119,119, -765,765,765,765,765,765,765,765,765,765,765,765,765,765,765,765, -765,765,765,765,765,765,765,765,765,119,119,119,119,119,119,119, -766,766,766,766,766,766,766,766,766,766,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +771,771,771,771,771,771,771,771,771,771,771,771,771,771,771,771, +771,771,771,771,771,772,772,772,772,772,772,772,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +773,773,773,773,773,773,773,773,773,773,773,773,773,773,773,773, +773,773,773,773,773,773,773,120,120,120,120,120,120,120,120,120, /* block 166 */ -767,767,767,768,768,768,768,768,768,768,768,768,768,768,768,768, -768,768,768,768,768,768,768,768,768,768,768,768,768,768,768,768, -768,768,768,768,768,768,768,767,767,767,767,767,769,767,767,767, -767,767,767,767,767,119,770,770,770,770,770,770,770,770,770,770, -771,771,771,771,768,769,769,119,119,119,119,119,119,119,119,119, -772,772,772,772,772,772,772,772,772,772,772,772,772,772,772,772, -772,772,772,772,772,772,772,772,772,772,772,772,772,772,772,772, -772,772,772,773,774,774,772,119,119,119,119,119,119,119,119,119, +774,775,774,776,776,776,776,776,776,776,776,776,776,776,776,776, +776,776,776,776,776,776,776,776,776,776,776,776,776,776,776,776, +776,776,776,776,776,776,776,776,776,776,776,776,776,776,776,776, +776,776,776,776,776,776,776,776,775,775,775,775,775,775,775,775, +775,775,775,775,775,775,775,777,777,777,777,777,777,777,120,120, +120,120,778,778,778,778,778,778,778,778,778,778,778,778,778,778, +778,778,778,778,778,778,779,779,779,779,779,779,779,779,779,779, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,775, /* block 167 */ -775,775,776,777,777,777,777,777,777,777,777,777,777,777,777,777, -777,777,777,777,777,777,777,777,777,777,777,777,777,777,777,777, -777,777,777,777,777,777,777,777,777,777,777,777,777,777,777,777, -777,777,777,776,776,776,775,775,775,775,775,775,775,775,775,776, -776,777,778,778,777,779,779,779,779,775,775,775,775,779,119,119, -780,780,780,780,780,780,780,780,780,780,777,779,777,779,779,779, -119,781,781,781,781,781,781,781,781,781,781,781,781,781,781,781, -781,781,781,781,781,119,119,119,119,119,119,119,119,119,119,119, +780,780,781,782,782,782,782,782,782,782,782,782,782,782,782,782, +782,782,782,782,782,782,782,782,782,782,782,782,782,782,782,782, +782,782,782,782,782,782,782,782,782,782,782,782,782,782,782,782, +781,781,781,780,780,780,780,781,781,780,780,783,783,784,783,783, +783,783,120,120,120,120,120,120,120,120,120,120,120,784,120,120, +785,785,785,785,785,785,785,785,785,785,785,785,785,785,785,785, +785,785,785,785,785,785,785,785,785,120,120,120,120,120,120,120, +786,786,786,786,786,786,786,786,786,786,120,120,120,120,120,120, /* block 168 */ -782,782,782,782,782,782,782,782,782,782,782,782,782,782,782,782, -782,782,119,782,782,782,782,782,782,782,782,782,782,782,782,782, -782,782,782,782,782,782,782,782,782,782,782,782,783,783,783,784, -784,784,783,783,784,783,784,784,785,785,785,785,785,785,784,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +787,787,787,788,788,788,788,788,788,788,788,788,788,788,788,788, +788,788,788,788,788,788,788,788,788,788,788,788,788,788,788,788, +788,788,788,788,788,788,788,787,787,787,787,787,789,787,787,787, +787,787,787,787,787,120,790,790,790,790,790,790,790,790,790,790, +791,791,791,791,788,789,789,788,120,120,120,120,120,120,120,120, +792,792,792,792,792,792,792,792,792,792,792,792,792,792,792,792, +792,792,792,792,792,792,792,792,792,792,792,792,792,792,792,792, +792,792,792,793,794,794,792,120,120,120,120,120,120,120,120,120, /* block 169 */ -786,786,786,786,786,786,786,119,786,119,786,786,786,786,119,786, -786,786,786,786,786,786,786,786,786,786,786,786,786,786,119,786, -786,786,786,786,786,786,786,786,786,787,119,119,119,119,119,119, -788,788,788,788,788,788,788,788,788,788,788,788,788,788,788,788, -788,788,788,788,788,788,788,788,788,788,788,788,788,788,788,788, -788,788,788,788,788,788,788,788,788,788,788,788,788,788,788,789, -790,790,790,789,789,789,789,789,789,789,789,119,119,119,119,119, -791,791,791,791,791,791,791,791,791,791,119,119,119,119,119,119, +795,795,796,797,797,797,797,797,797,797,797,797,797,797,797,797, +797,797,797,797,797,797,797,797,797,797,797,797,797,797,797,797, +797,797,797,797,797,797,797,797,797,797,797,797,797,797,797,797, +797,797,797,796,796,796,795,795,795,795,795,795,795,795,795,796, +796,797,798,798,797,799,799,799,799,795,795,795,795,799,796,795, +800,800,800,800,800,800,800,800,800,800,797,799,797,799,799,799, +120,801,801,801,801,801,801,801,801,801,801,801,801,801,801,801, +801,801,801,801,801,120,120,120,120,120,120,120,120,120,120,120, /* block 170 */ -792,793,794,795,119,796,796,796,796,796,796,796,796,119,119,796, -796,119,119,796,796,796,796,796,796,796,796,796,796,796,796,796, -796,796,796,796,796,796,796,796,796,119,796,796,796,796,796,796, -796,119,796,796,119,796,796,796,796,796,119,797,793,796,798,794, -792,794,794,794,794,119,119,794,794,119,119,794,794,794,119,119, -796,119,119,119,119,119,119,798,119,119,119,119,119,796,796,796, -796,796,794,794,119,119,792,792,792,792,792,792,792,119,119,119, -792,792,792,792,792,119,119,119,119,119,119,119,119,119,119,119, +802,802,802,802,802,802,802,802,802,802,802,802,802,802,802,802, +802,802,120,802,802,802,802,802,802,802,802,802,802,802,802,802, +802,802,802,802,802,802,802,802,802,802,802,802,803,803,803,804, +804,804,803,803,804,803,804,804,805,805,805,805,805,805,804,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 171 */ -799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799, -799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799, -799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799, -799,799,799,799,799,800,800,800,801,801,801,801,801,801,801,801, -800,800,801,801,801,800,801,799,799,799,799,802,802,802,802,802, -803,803,803,803,803,803,803,803,803,803,119,802,119,802,801,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +806,806,806,806,806,806,806,120,806,120,806,806,806,806,120,806, +806,806,806,806,806,806,806,806,806,806,806,806,806,806,120,806, +806,806,806,806,806,806,806,806,806,807,120,120,120,120,120,120, +808,808,808,808,808,808,808,808,808,808,808,808,808,808,808,808, +808,808,808,808,808,808,808,808,808,808,808,808,808,808,808,808, +808,808,808,808,808,808,808,808,808,808,808,808,808,808,808,809, +810,810,810,809,809,809,809,809,809,809,809,120,120,120,120,120, +811,811,811,811,811,811,811,811,811,811,120,120,120,120,120,120, /* block 172 */ -804,804,804,804,804,804,804,804,804,804,804,804,804,804,804,804, -804,804,804,804,804,804,804,804,804,804,804,804,804,804,804,804, -804,804,804,804,804,804,804,804,804,804,804,804,804,804,804,804, -805,806,806,807,807,807,807,807,807,806,807,806,806,805,806,807, -807,806,807,807,804,804,808,804,119,119,119,119,119,119,119,119, -809,809,809,809,809,809,809,809,809,809,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +812,813,814,815,120,816,816,816,816,816,816,816,816,120,120,816, +816,120,120,816,816,816,816,816,816,816,816,816,816,816,816,816, +816,816,816,816,816,816,816,816,816,120,816,816,816,816,816,816, +816,120,816,816,120,816,816,816,816,816,120,817,813,816,818,814, +812,814,814,814,814,120,120,814,814,120,120,814,814,814,120,120, +816,120,120,120,120,120,120,818,120,120,120,120,120,816,816,816, +816,816,814,814,120,120,812,812,812,812,812,812,812,120,120,120, +812,812,812,812,812,120,120,120,120,120,120,120,120,120,120,120, /* block 173 */ -810,810,810,810,810,810,810,810,810,810,810,810,810,810,810,810, -810,810,810,810,810,810,810,810,810,810,810,810,810,810,810,810, -810,810,810,810,810,810,810,810,810,810,810,810,810,810,810,811, -812,812,813,813,813,813,119,119,812,812,812,812,813,813,812,813, -813,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814, -814,814,814,814,814,814,814,814,810,810,810,810,813,813,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +819,819,819,819,819,819,819,819,819,819,819,819,819,819,819,819, +819,819,819,819,819,819,819,819,819,819,819,819,819,819,819,819, +819,819,819,819,819,819,819,819,819,819,819,819,819,819,819,819, +819,819,819,819,819,820,820,820,821,821,821,821,821,821,821,821, +820,820,821,821,821,820,821,819,819,819,819,822,822,822,822,822, +823,823,823,823,823,823,823,823,823,823,822,822,120,822,821,819, +819,819,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 174 */ -815,815,815,815,815,815,815,815,815,815,815,815,815,815,815,815, -815,815,815,815,815,815,815,815,815,815,815,815,815,815,815,815, -815,815,815,815,815,815,815,815,815,815,815,815,815,815,815,815, -816,816,816,817,817,817,817,817,817,817,817,816,816,817,816,817, -817,818,818,818,815,119,119,119,119,119,119,119,119,119,119,119, -819,819,819,819,819,819,819,819,819,819,119,119,119,119,119,119, -392,392,392,392,392,392,392,392,392,392,392,392,392,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +824,824,824,824,824,824,824,824,824,824,824,824,824,824,824,824, +824,824,824,824,824,824,824,824,824,824,824,824,824,824,824,824, +824,824,824,824,824,824,824,824,824,824,824,824,824,824,824,824, +825,826,826,827,827,827,827,827,827,826,827,826,826,825,826,827, +827,826,827,827,824,824,828,824,120,120,120,120,120,120,120,120, +829,829,829,829,829,829,829,829,829,829,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 175 */ -820,820,820,820,820,820,820,820,820,820,820,820,820,820,820,820, -820,820,820,820,820,820,820,820,820,820,820,820,820,820,820,820, -820,820,820,820,820,820,820,820,820,820,820,821,822,821,822,822, -821,821,821,821,821,821,822,821,119,119,119,119,119,119,119,119, -823,823,823,823,823,823,823,823,823,823,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +830,830,830,830,830,830,830,830,830,830,830,830,830,830,830,830, +830,830,830,830,830,830,830,830,830,830,830,830,830,830,830,830, +830,830,830,830,830,830,830,830,830,830,830,830,830,830,830,831, +832,832,833,833,833,833,120,120,832,832,832,832,833,833,832,833, +833,834,834,834,834,834,834,834,834,834,834,834,834,834,834,834, +834,834,834,834,834,834,834,834,830,830,830,830,833,833,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 176 */ -824,824,824,824,824,824,824,824,824,824,824,824,824,824,824,824, -824,824,824,824,824,824,824,824,824,824,824,119,119,825,825,825, -826,826,825,825,825,825,826,825,825,825,825,825,119,119,119,119, -827,827,827,827,827,827,827,827,827,827,828,828,829,829,829,830, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +835,835,835,835,835,835,835,835,835,835,835,835,835,835,835,835, +835,835,835,835,835,835,835,835,835,835,835,835,835,835,835,835, +835,835,835,835,835,835,835,835,835,835,835,835,835,835,835,835, +836,836,836,837,837,837,837,837,837,837,837,836,836,837,836,837, +837,838,838,838,835,120,120,120,120,120,120,120,120,120,120,120, +839,839,839,839,839,839,839,839,839,839,120,120,120,120,120,120, +394,394,394,394,394,394,394,394,394,394,394,394,394,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 177 */ -831,831,831,831,831,831,831,831,831,831,831,831,831,831,831,831, -831,831,831,831,831,831,831,831,831,831,831,831,831,831,831,831, -831,831,831,831,831,831,831,831,831,831,831,831,832,832,832,833, -833,833,833,833,833,833,833,833,832,833,833,834,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +840,840,840,840,840,840,840,840,840,840,840,840,840,840,840,840, +840,840,840,840,840,840,840,840,840,840,840,840,840,840,840,840, +840,840,840,840,840,840,840,840,840,840,840,841,842,841,842,842, +841,841,841,841,841,841,842,841,840,120,120,120,120,120,120,120, +843,843,843,843,843,843,843,843,843,843,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 178 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -835,835,835,835,835,835,835,835,835,835,835,835,835,835,835,835, -835,835,835,835,835,835,835,835,835,835,835,835,835,835,835,835, -836,836,836,836,836,836,836,836,836,836,836,836,836,836,836,836, -836,836,836,836,836,836,836,836,836,836,836,836,836,836,836,836, -837,837,837,837,837,837,837,837,837,837,838,838,838,838,838,838, -838,838,838,119,119,119,119,119,119,119,119,119,119,119,119,839, +844,844,844,844,844,844,844,844,844,844,844,844,844,844,844,844, +844,844,844,844,844,844,844,844,844,844,844,120,120,845,845,845, +846,846,845,845,845,845,846,845,845,845,845,845,120,120,120,120, +847,847,847,847,847,847,847,847,847,847,848,848,849,849,849,850, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 179 */ -840,841,841,841,841,841,841,841,841,841,841,840,840,840,840,840, -840,840,840,840,840,840,840,840,840,840,840,840,840,840,840,840, -840,840,840,840,840,840,840,840,840,840,840,840,840,840,840,840, -840,840,840,841,841,841,841,841,841,842,843,841,841,841,841,844, -844,844,844,844,844,844,844,841,119,119,119,119,119,119,119,119, -845,846,846,846,846,846,846,847,847,846,846,846,845,845,845,845, -845,845,845,845,845,845,845,845,845,845,845,845,845,845,845,845, -845,845,845,845,845,845,845,845,845,845,845,845,845,845,845,845, +851,851,851,851,851,851,851,851,851,851,851,851,851,851,851,851, +851,851,851,851,851,851,851,851,851,851,851,851,851,851,851,851, +851,851,851,851,851,851,851,851,851,851,851,851,852,852,852,853, +853,853,853,853,853,853,853,853,852,853,853,854,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 180 */ -845,845,845,845,119,119,848,848,848,848,846,846,846,846,846,846, -846,846,846,846,846,846,846,847,846,846,849,849,849,845,849,849, -849,849,849,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -850,850,850,850,850,850,850,850,850,850,850,850,850,850,850,850, -850,850,850,850,850,850,850,850,850,850,850,850,850,850,850,850, -850,850,850,850,850,850,850,850,850,850,850,850,850,850,850,850, -850,850,850,850,850,850,850,850,850,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +855,855,855,855,855,855,855,855,855,855,855,855,855,855,855,855, +855,855,855,855,855,855,855,855,855,855,855,855,855,855,855,855, +856,856,856,856,856,856,856,856,856,856,856,856,856,856,856,856, +856,856,856,856,856,856,856,856,856,856,856,856,856,856,856,856, +857,857,857,857,857,857,857,857,857,857,858,858,858,858,858,858, +858,858,858,120,120,120,120,120,120,120,120,120,120,120,120,859, /* block 181 */ -851,851,851,851,851,851,851,851,851,119,851,851,851,851,851,851, -851,851,851,851,851,851,851,851,851,851,851,851,851,851,851,851, -851,851,851,851,851,851,851,851,851,851,851,851,851,851,851,852, -853,853,853,853,853,853,853,119,853,853,853,853,853,853,852,853, -851,854,854,854,854,854,119,119,119,119,119,119,119,119,119,119, -855,855,855,855,855,855,855,855,855,855,856,856,856,856,856,856, -856,856,856,856,856,856,856,856,856,856,856,856,856,119,119,119, -857,857,858,858,858,858,858,858,858,858,858,858,858,858,858,858, +860,860,860,860,860,860,860,120,120,860,120,120,860,860,860,860, +860,860,860,860,120,860,860,120,860,860,860,860,860,860,860,860, +860,860,860,860,860,860,860,860,860,860,860,860,860,860,860,860, +861,862,862,862,862,862,120,862,862,120,120,863,863,862,863,864, +862,864,862,863,865,865,865,120,120,120,120,120,120,120,120,120, +866,866,866,866,866,866,866,866,866,866,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 182 */ -858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858, -119,119,859,859,859,859,859,859,859,859,859,859,859,859,859,859, -859,859,859,859,859,859,859,859,119,860,859,859,859,859,859,859, -859,860,859,859,860,859,859,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +867,867,867,867,867,867,867,867,120,120,867,867,867,867,867,867, +867,867,867,867,867,867,867,867,867,867,867,867,867,867,867,867, +867,867,867,867,867,867,867,867,867,867,867,867,867,867,867,867, +867,868,868,868,869,869,869,869,120,120,869,869,868,868,868,868, +869,867,870,867,868,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 183 */ -861,861,861,861,861,861,861,119,861,861,119,861,861,861,861,861, -861,861,861,861,861,861,861,861,861,861,861,861,861,861,861,861, -861,861,861,861,861,861,861,861,861,861,861,861,861,861,861,861, -861,862,862,862,862,862,862,119,119,119,862,119,862,862,119,862, -862,862,862,862,862,862,863,862,119,119,119,119,119,119,119,119, -864,864,864,864,864,864,864,864,864,864,119,119,119,119,119,119, -865,865,865,865,865,865,119,865,865,119,865,865,865,865,865,865, -865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865, +871,872,872,872,872,872,872,872,872,872,872,871,871,871,871,871, +871,871,871,871,871,871,871,871,871,871,871,871,871,871,871,871, +871,871,871,871,871,871,871,871,871,871,871,871,871,871,871,871, +871,871,871,872,872,872,872,872,872,873,874,872,872,872,872,875, +875,875,875,875,875,875,875,872,120,120,120,120,120,120,120,120, +876,877,877,877,877,877,877,878,878,877,877,877,876,876,876,876, +876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, +876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, /* block 184 */ -865,865,865,865,865,865,865,865,865,865,866,866,866,866,866,119, -867,867,119,866,866,867,866,867,865,119,119,119,119,119,119,119, -868,868,868,868,868,868,868,868,868,868,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +876,876,876,876,879,879,879,879,879,879,877,877,877,877,877,877, +877,877,877,877,877,877,877,878,877,877,880,880,880,876,880,880, +880,880,880,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +881,881,881,881,881,881,881,881,881,881,881,881,881,881,881,881, +881,881,881,881,881,881,881,881,881,881,881,881,881,881,881,881, +881,881,881,881,881,881,881,881,881,881,881,881,881,881,881,881, +881,881,881,881,881,881,881,881,881,120,120,120,120,120,120,120, /* block 185 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -869,869,869,869,869,869,869,869,869,869,869,869,869,869,869,869, -869,869,869,870,870,871,871,872,872,119,119,119,119,119,119,119, +882,882,882,882,882,882,882,882,882,120,882,882,882,882,882,882, +882,882,882,882,882,882,882,882,882,882,882,882,882,882,882,882, +882,882,882,882,882,882,882,882,882,882,882,882,882,882,882,883, +884,884,884,884,884,884,884,120,884,884,884,884,884,884,883,884, +882,885,885,885,885,885,120,120,120,120,120,120,120,120,120,120, +886,886,886,886,886,886,886,886,886,886,887,887,887,887,887,887, +887,887,887,887,887,887,887,887,887,887,887,887,887,120,120,120, +888,888,889,889,889,889,889,889,889,889,889,889,889,889,889,889, /* block 186 */ -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, +889,889,889,889,889,889,889,889,889,889,889,889,889,889,889,889, +120,120,890,890,890,890,890,890,890,890,890,890,890,890,890,890, +890,890,890,890,890,890,890,890,120,891,890,890,890,890,890,890, +890,891,890,890,891,890,890,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 187 */ -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +892,892,892,892,892,892,892,120,892,892,120,892,892,892,892,892, +892,892,892,892,892,892,892,892,892,892,892,892,892,892,892,892, +892,892,892,892,892,892,892,892,892,892,892,892,892,892,892,892, +892,893,893,893,893,893,893,120,120,120,893,120,893,893,120,893, +893,893,893,893,893,893,894,893,120,120,120,120,120,120,120,120, +895,895,895,895,895,895,895,895,895,895,120,120,120,120,120,120, +896,896,896,896,896,896,120,896,896,120,896,896,896,896,896,896, +896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896, /* block 188 */ -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,874, -874,874,874,874,874,874,874,874,874,874,874,874,874,874,874,119, -875,875,875,875,875,119,119,119,119,119,119,119,119,119,119,119, +896,896,896,896,896,896,896,896,896,896,897,897,897,897,897,120, +898,898,120,897,897,898,897,898,896,120,120,120,120,120,120,120, +899,899,899,899,899,899,899,899,899,899,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 189 */ -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,873,873,873,873,873,873,873,873,873,873,873,873, -873,873,873,873,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +900,900,900,900,900,900,900,900,900,900,900,900,900,900,900,900, +900,900,900,901,901,902,902,903,903,120,120,120,120,120,120,120, /* block 190 */ -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +590,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +904,904,904,904,904,904,904,904,904,904,904,904,904,904,904,904, +293,293,904,293,904,295,295,295,295,295,295,295,295,296,296,296, +296,295,295,295,295,295,295,295,295,295,295,295,295,295,295,295, +295,295,120,120,120,120,120,120,120,120,120,120,120,120,120,905, /* block 191 */ -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,876, -876,876,876,876,876,876,876,876,876,876,876,876,876,876,876,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, /* block 192 */ -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 193 */ -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,877,877,877,877,877,877,877,877,877, -877,877,877,877,877,877,877,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,907, +907,907,907,907,907,907,907,907,907,907,907,907,907,907,907,120, +908,908,908,908,908,120,120,120,120,120,120,120,120,120,120,120, /* block 194 */ -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,906,906,906,906,906,906,906,906,906,906,906,906, +906,906,906,906,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 195 */ -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,592,592,592,592,592,592,592, -592,592,592,592,592,592,592,592,592,119,119,119,119,119,119,119, -878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878, -878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,119, -879,879,879,879,879,879,879,879,879,879,119,119,119,119,880,880, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, /* block 196 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -881,881,881,881,881,881,881,881,881,881,881,881,881,881,881,881, -881,881,881,881,881,881,881,881,881,881,881,881,881,881,119,119, -882,882,882,882,882,883,119,119,119,119,119,119,119,119,119,119, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,909, +909,909,909,909,909,909,909,909,909,909,909,909,909,909,909,120, +910,910,910,910,910,910,910,910,910,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 197 */ -884,884,884,884,884,884,884,884,884,884,884,884,884,884,884,884, -884,884,884,884,884,884,884,884,884,884,884,884,884,884,884,884, -884,884,884,884,884,884,884,884,884,884,884,884,884,884,884,884, -885,885,885,885,885,885,885,886,886,886,886,886,887,887,887,887, -888,888,888,888,886,887,119,119,119,119,119,119,119,119,119,119, -889,889,889,889,889,889,889,889,889,889,119,890,890,890,890,890, -890,890,119,884,884,884,884,884,884,884,884,884,884,884,884,884, -884,884,884,884,884,884,884,884,119,119,119,119,119,884,884,884, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, /* block 198 */ -884,884,884,884,884,884,884,884,884,884,884,884,884,884,884,884, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +911,911,911,911,911,911,911,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 199 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -891,891,891,891,891,891,891,891,891,891,891,891,891,891,891,891, -891,891,891,891,891,891,891,891,891,891,891,891,891,891,891,891, -892,892,892,892,892,892,892,892,892,892,892,892,892,892,892,892, -892,892,892,892,892,892,892,892,892,892,892,892,892,892,892,892, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, /* block 200 */ -893,893,893,893,893,893,893,893,893,893,893,893,893,893,893,893, -893,893,893,893,893,893,893,894,894,894,894,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,601,601,601,601,601,601,601, +601,601,601,601,601,601,601,601,601,120,120,120,120,120,120,120, +912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, +912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,120, +913,913,913,913,913,913,913,913,913,913,120,120,120,120,914,914, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 201 */ -895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895, -895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895, -895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895, -895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895, -895,895,895,895,895,119,119,119,119,119,119,119,119,119,119,119, -895,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896, -896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896, -896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, +915,915,915,915,915,915,915,915,915,915,915,915,915,915,120,120, +916,916,916,916,916,917,120,120,120,120,120,120,120,120,120,120, /* block 202 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,897, -897,897,897,898,898,898,898,898,898,898,898,898,898,898,898,898, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -899,900,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, +918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, +918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, +919,919,919,919,919,919,919,920,920,920,920,920,921,921,921,921, +922,922,922,922,920,921,120,120,120,120,120,120,120,120,120,120, +923,923,923,923,923,923,923,923,923,923,120,924,924,924,924,924, +924,924,120,918,918,918,918,918,918,918,918,918,918,918,918,918, +918,918,918,918,918,918,918,918,120,120,120,120,120,918,918,918, /* block 203 */ -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, +918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 204 */ -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +925,925,925,925,925,925,925,925,925,925,925,925,925,925,925,925, +925,925,925,925,925,925,925,925,925,925,925,925,925,925,925,925, +926,926,926,926,926,926,926,926,926,926,926,926,926,926,926,926, +926,926,926,926,926,926,926,926,926,926,926,926,926,926,926,926, /* block 205 */ -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,901,901,901,901,901,901,901,901,901,901,901,901,901, -901,901,901,119,119,119,119,119,119,119,119,119,119,119,119,119, +927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, +927,927,927,927,927,927,927,928,928,928,928,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 206 */ -569,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, +929,929,929,929,929,929,929,929,929,929,929,929,929,929,929,929, +929,929,929,929,929,929,929,929,929,929,929,929,929,929,929,929, +929,929,929,929,929,929,929,929,929,929,929,929,929,929,929,929, +929,929,929,929,929,929,929,929,929,929,929,929,929,929,929,929, +929,929,929,929,929,929,929,929,929,929,929,120,120,120,120,930, +929,931,931,931,931,931,931,931,931,931,931,931,931,931,931,931, +931,931,931,931,931,931,931,931,931,931,931,931,931,931,931,931, +931,931,931,931,931,931,931,931,931,931,931,931,931,931,931,931, /* block 207 */ -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, +931,931,931,931,931,931,931,931,120,120,120,120,120,120,120,930, +930,930,930,932,932,932,932,932,932,932,932,932,932,932,932,932, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +933,934, 5,111,935,120,120,120,120,120,120,120,120,120,120,120, +936,936,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 208 */ -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,564, -564,564,564,564,564,564,564,564,564,564,564,564,564,564,564,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, /* block 209 */ -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,937,937,937,937,937,937,937,937, +937,937,937,937,937,937,937,937,120,120,120,120,120,120,120,120, /* block 210 */ -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,902,902,902,902, -902,902,902,902,902,902,902,902,902,902,902,902,119,119,119,119, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, /* block 211 */ -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903, -903,903,903,903,903,903,903,903,903,903,903,119,119,119,119,119, -903,903,903,903,903,903,903,903,903,903,903,903,903,119,119,119, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,938,938,938,938,938,938,938,938,938,938, +938,938,938,938,938,938,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 212 */ -903,903,903,903,903,903,903,903,903,119,119,119,119,119,119,119, -903,903,903,903,903,903,903,903,903,903,119,119,904,905,905,906, -907,907,907,907,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +937,937,937,937,937,937,937,937,937,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, /* block 213 */ +578,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, + +/* block 214 */ +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, + +/* block 215 */ +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, +573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +573,573,573,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,578,578,578,578,120,120,120,120,120,120,120,120, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, + +/* block 216 */ +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, + +/* block 217 */ +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,939,939,939,939, +939,939,939,939,939,939,939,939,939,939,939,939,120,120,120,120, + +/* block 218 */ +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,940,940,940,940,940, +940,940,940,940,940,940,940,940,940,940,940,120,120,120,120,120, +940,940,940,940,940,940,940,940,940,940,940,940,940,120,120,120, + +/* block 219 */ +940,940,940,940,940,940,940,940,940,120,120,120,120,120,120,120, +940,940,940,940,940,940,940,940,940,940,120,120,941,942,942,943, +944,944,944,944,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 220 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, @@ -3802,309 +3926,339 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20,119,119,119,119,119,119,119,119,119,119, + 20, 20, 20, 20, 20, 20,120,120,120,120,120,120,120,120,120,120, -/* block 214 */ +/* block 221 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20,119,119, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20,120,120, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20,908,909,112,112,112, 20, 20, 20,909,908,908, -908,908,908, 24, 24, 24, 24, 24, 24, 24, 24,112,112,112,112,112, + 20, 20, 20, 20, 20,945,946,113,113,113, 20, 20, 20,946,945,945, +945,945,945, 24, 24, 24, 24, 24, 24, 24, 24,113,113,113,113,113, -/* block 215 */ -112,112,112, 20, 20,112,112,112,112,112,112,112, 20, 20, 20, 20, +/* block 222 */ +113,113,113, 20, 20,113,113,113,113,113,113,113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,112,112,112,112, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,113,113,113,113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, + 20, 20, 20, 20, 20, 20, 20, 20, 20,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, -/* block 216 */ -671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, -671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, -671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, -671,671,671,671,671,671,671,671,671,671,671,671,671,671,671,671, -671,671,910,910,910,671,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +/* block 223 */ +685,685,685,685,685,685,685,685,685,685,685,685,685,685,685,685, +685,685,685,685,685,685,685,685,685,685,685,685,685,685,685,685, +685,685,685,685,685,685,685,685,685,685,685,685,685,685,685,685, +685,685,685,685,685,685,685,685,685,685,685,685,685,685,685,685, +685,685,947,947,947,685,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, -/* block 217 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +/* block 224 */ +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25,119,119,119,119,119,119,119,119,119,119,119,119, + 25, 25, 25, 25,120,120,120,120,120,120,120,120,120,120,120,120, -/* block 218 */ +/* block 225 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20,119,119,119,119,119,119,119,119,119, -573,573,573,573,573,573,573,573,573,573,573,573,573,573,573,573, -573,573, 25, 25, 25, 25, 25, 25, 25,119,119,119,119,119,119,119, - -/* block 219 */ -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,505,505, -505,505,505,505,505,119,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, - -/* block 220 */ -504,504,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,504,119,504,504, -119,119,504,119,119,504,504,119,119,504,504,504,504,119,504,504, -504,504,504,504,504,504,505,505,505,505,119,505,119,505,505,505, -505,505,505,505,119,505,505,505,505,505,505,505,505,505,505,505, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, - -/* block 221 */ -505,505,505,505,504,504,119,504,504,504,504,119,119,504,504,504, -504,504,504,504,504,119,504,504,504,504,504,504,504,119,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,504,504,119,504,504,504,504,119, -504,504,504,504,504,119,504,119,119,119,504,504,504,504,504,504, -504,119,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, - -/* block 222 */ -504,504,504,504,504,504,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, - -/* block 223 */ -505,505,505,505,505,505,505,505,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, - -/* block 224 */ -504,504,504,504,504,504,504,504,504,504,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,119,119,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504, 9,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505, 9,505,505,505,505, -505,505,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504, 9,505,505,505,505, - -/* block 225 */ -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505, 9,505,505,505,505,505,505,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504, 9,505,505,505,505,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, 9, -505,505,505,505,505,505,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, 9, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, + 20, 20, 20, 20, 20, 20, 20,120,120,120,120,120,120,120,120,120, +582,582,582,582,582,582,582,582,582,582,582,582,582,582,582,582, +582,582, 25, 25, 25, 25, 25, 25, 25,120,120,120,120,120,120,120, /* block 226 */ -505,505,505,505,505,505,505,505,505, 9,505,505,505,505,505,505, -504,504,504,504,504,504,504,504,504,504,504,504,504,504,504,504, -504,504,504,504,504,504,504,504,504, 9,505,505,505,505,505,505, -505,505,505,505,505,505,505,505,505,505,505,505,505,505,505,505, -505,505,505, 9,505,505,505,505,505,505,504,505,119,119, 11, 11, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,514,514, +514,514,514,514,514,120,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, /* block 227 */ -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, -911,911,911,911,911,911,911,911,911,911,911,911,911,911,911,911, +513,513,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,513,120,513,513, +120,120,513,120,120,513,513,120,120,513,513,513,513,120,513,513, +513,513,513,513,513,513,514,514,514,514,120,514,120,514,514,514, +514,514,514,514,120,514,514,514,514,514,514,514,514,514,514,514, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, /* block 228 */ -912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -912,912,912,912,912,912,912,911,911,911,911,912,912,912,912,912, -912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -912,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -912,912,912,912,912,912,912,912,912,912,912,912,912,911,911,911, -911,911,911,911,911,912,911,911,911,911,911,911,911,911,911,911, +514,514,514,514,513,513,120,513,513,513,513,120,120,513,513,513, +513,513,513,513,513,120,513,513,513,513,513,513,513,120,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,513,513,120,513,513,513,513,120, +513,513,513,513,513,120,513,120,120,120,513,513,513,513,513,513, +513,120,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, /* block 229 */ -911,911,911,911,912,911,911,913,913,913,913,913,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,912,912,912,912,912, -119,912,912,912,912,912,912,912,912,912,912,912,912,912,912,912, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +513,513,513,513,513,513,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, /* block 230 */ -914,914,914,914,914,914,914,119,914,914,914,914,914,914,914,914, -914,914,914,914,914,914,914,914,914,119,119,914,914,914,914,914, -914,914,119,914,914,119,914,914,914,914,914,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +514,514,514,514,514,514,514,514,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, /* block 231 */ -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, +513,513,513,513,513,513,513,513,513,513,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,120,120,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513, 9,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514, 9,514,514,514,514, +514,514,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513, 9,514,514,514,514, /* block 232 */ -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915, -915,915,915,915,915,119,119,916,916,916,916,916,916,916,916,916, -917,917,917,917,917,917,917,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514, 9,514,514,514,514,514,514,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513, 9,514,514,514,514,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, 9, +514,514,514,514,514,514,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, 9, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, /* block 233 */ -918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, -918,918,918,918,918,918,918,918,918,918,918,918,918,918,918,918, -918,918,919,919,919,919,919,919,919,919,919,919,919,919,919,919, -919,919,919,919,919,919,919,919,919,919,919,919,919,919,919,919, -919,919,919,919,920,920,920,920,920,920,920,119,119,119,119,119, -921,921,921,921,921,921,921,921,921,921,119,119,119,119,922,922, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +514,514,514,514,514,514,514,514,514, 9,514,514,514,514,514,514, +513,513,513,513,513,513,513,513,513,513,513,513,513,513,513,513, +513,513,513,513,513,513,513,513,513, 9,514,514,514,514,514,514, +514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514, +514,514,514, 9,514,514,514,514,514,514,513,514,120,120, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, /* block 234 */ -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, +948,948,948,948,948,948,948,948,948,948,948,948,948,948,948,948, /* block 235 */ +949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +949,949,949,949,949,949,949,948,948,948,948,949,949,949,949,949, +949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +949,949,949,949,949,949,949,949,949,949,949,949,949,948,948,948, +948,948,948,948,948,949,948,948,948,948,948,948,948,948,948,948, + +/* block 236 */ +948,948,948,948,949,948,948,950,950,950,950,950,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,949,949,949,949,949, +120,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 237 */ +951,951,951,951,951,951,951,120,951,951,951,951,951,951,951,951, +951,951,951,951,951,951,951,951,951,120,120,951,951,951,951,951, +951,951,120,951,951,120,951,951,951,951,951,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 238 */ +952,952,952,952,952,952,952,952,952,952,952,952,952,952,952,952, +952,952,952,952,952,952,952,952,952,952,952,952,952,952,952,952, +952,952,952,952,952,952,952,952,952,952,952,952,952,120,120,120, +953,953,953,953,953,953,953,954,954,954,954,954,954,954,120,120, +955,955,955,955,955,955,955,955,955,955,120,120,120,120,952,956, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 239 */ +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +957,957,957,957,957,957,957,957,957,957,957,957,957,957,957,957, +957,957,957,957,957,957,957,957,957,957,957,957,957,957,957,957, +957,957,957,957,957,957,957,957,957,957,957,957,958,958,958,958, +959,959,959,959,959,959,959,959,959,959,120,120,120,120,120,960, + +/* block 240 */ +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, + +/* block 241 */ +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961, +961,961,961,961,961,120,120,962,962,962,962,962,962,962,962,962, +963,963,963,963,963,963,963,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 242 */ +964,964,964,964,964,964,964,964,964,964,964,964,964,964,964,964, +964,964,964,964,964,964,964,964,964,964,964,964,964,964,964,964, +964,964,965,965,965,965,965,965,965,965,965,965,965,965,965,965, +965,965,965,965,965,965,965,965,965,965,965,965,965,965,965,965, +965,965,965,965,966,966,966,966,966,966,966,967,120,120,120,120, +968,968,968,968,968,968,968,968,968,968,120,120,120,120,969,969, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 243 */ +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + +/* block 244 */ 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 20, 25, 25, 25, - 6, 25, 25, 25, 25,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, + 6, 25, 25, 25, 25,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, -/* block 236 */ -224,224,224,224,119,224,224,224,224,224,224,224,224,224,224,224, +/* block 245 */ +120, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 20, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 246 */ +224,224,224,224,120,224,224,224,224,224,224,224,224,224,224,224, 224,224,224,224,224,224,224,224,224,224,224,224,224,224,224,224, -119,224,224,119,224,119,119,224,119,224,224,224,224,224,224,224, -224,224,224,119,224,224,224,224,119,224,119,224,119,119,119,119, -119,119,224,119,119,119,119,224,119,224,119,224,119,224,224,224, -119,224,224,119,224,119,119,224,119,224,119,224,119,224,119,224, -119,224,224,119,224,119,119,224,224,224,224,119,224,224,224,224, -224,224,224,119,224,224,224,224,119,224,224,224,224,119,224,119, +120,224,224,120,224,120,120,224,120,224,224,224,224,224,224,224, +224,224,224,120,224,224,224,224,120,224,120,224,120,120,120,120, +120,120,224,120,120,120,120,224,120,224,120,224,120,224,224,224, +120,224,224,120,224,120,120,224,120,224,120,224,120,224,120,224, +120,224,224,120,224,120,120,224,224,224,224,120,224,224,224,224, +224,224,224,120,224,224,224,224,120,224,224,224,224,120,224,120, -/* block 237 */ -224,224,224,224,224,224,224,224,224,224,119,224,224,224,224,224, -224,224,224,224,224,224,224,224,224,224,224,224,119,119,119,119, -119,224,224,224,119,224,224,224,224,224,119,224,224,224,224,224, -224,224,224,224,224,224,224,224,224,224,224,224,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -217,217,119,119,119,119,119,119,119,119,119,119,119,119,119,119, +/* block 247 */ +224,224,224,224,224,224,224,224,224,224,120,224,224,224,224,224, +224,224,224,224,224,224,224,224,224,224,224,224,120,120,120,120, +120,224,224,224,120,224,224,224,224,224,120,224,224,224,224,224, +224,224,224,224,224,224,224,224,224,224,224,224,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +217,217,120,120,120,120,120,120,120,120,120,120,120,120,120,120, -/* block 238 */ +/* block 248 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923,923,923,923, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970,970,970, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -/* block 239 */ +/* block 249 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21,923,923,923,923,923,923,923,923,923,923,923,923, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923, -923, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -923, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -923, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21,970,970,970,970,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970, +970, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, +970, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, +970, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21,923,923,923,923,923,923,923,923,923,923, + 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970,970,970, -/* block 240 */ - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,923,923,923, +/* block 250 */ + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 21, 21, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, -/* block 241 */ +/* block 251 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,924,924,924,924,924,924,924,924,924,924, -924,924,924,924,924,924,924,924,924,924,924,924,924,924,924,924, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,971,971,971,971,971,971,971,971,971,971, +971,971,971,971,971,971,971,971,971,971,971,971,971,971,971,971, -/* block 242 */ -925, 21, 21,923,923,923,923,923,923,923,923,923,923,923,923,923, +/* block 252 */ +972, 21, 21,970,970,970,970,970,970,970,970,970,970,970,970,970, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, - 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20,923,923,923,923, - 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923,923,923,923, -575,575,923,923,923,923,923,923,923,923,923,923,923,923,923,923, - 21, 21, 21, 21, 21, 21,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, + 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20,970,970,970,970, + 20, 20, 20, 20, 20, 20, 20, 20, 20,970,970,970,970,970,970,970, +584,584,970,970,970,970,970,970,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, -/* block 243 */ -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, +/* block 253 */ +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, -/* block 244 */ +/* block 254 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -4114,7 +4268,7 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -/* block 245 */ +/* block 255 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -4122,9 +4276,9 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,926,926,926,926,926, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,973,973,973,973,973, -/* block 246 */ +/* block 256 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -4134,7 +4288,7 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, -/* block 247 */ +/* block 257 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, @@ -4144,17 +4298,17 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, -/* block 248 */ +/* block 258 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21,923,923,923,923,923,923,923,923,923,923,923, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923,923,923, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923,923,923,923,923,923, + 21, 21, 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970,970, -/* block 249 */ +/* block 259 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, @@ -4162,187 +4316,217 @@ const uint16_t PRIV(ucd_stage2)[] = { /* 68608 bytes, block = 128 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20,923,923,923,923,923,923,923,923,923,923,923,923, + 20, 20, 20, 20,970,970,970,970,970,970,970,970,970,970,970,970, -/* block 250 */ +/* block 260 */ 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 21, 21, 21, 21,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, + 20, 20, 20, 20, 20, 21, 21, 21, 21,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, -/* block 251 */ - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923, +/* block 261 */ + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,970,970,970,970, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923,923,923,923,923, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923,923,923, + 20, 20, 20, 20, 20, 20, 20, 20,970,970,970,970,970,970,970,970, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,970,970,970,970,970,970, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, -/* block 252 */ - 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923,923,923,923,923, +/* block 262 */ + 20, 20, 20, 20, 20, 20, 20, 20,970,970,970,970,970,970,970,970, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,970,970, + 21, 21,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, -/* block 253 */ - 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,923,923,923,923, +/* block 263 */ + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20, 21, 21, 21,923, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21,923,923, 21, 21, 21, 21,923,923,923, 21,923, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21,970, 21, 21, 21, 21, 21, 21, -/* block 254 */ +/* block 264 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - 21, 21, 21,923,923,923,923,923,923,923,923,923,923,923,923,923, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923,923,923,923,923,923, - 21, 21, 21,923,923,923,923,923,923,923,923,923,923,923,923,923, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - -/* block 255 */ -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, - -/* block 256 */ -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923, -923,923,923,923,923,923,923,923,923,923,923,923,923,923,119,119, - -/* block 257 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, - -/* block 258 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,119,119,119,119,119,119,119,119,119,119,119, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, - -/* block 259 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,119,119, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, - -/* block 260 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, - -/* block 261 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, - -/* block 262 */ -577,577,577,577,577,577,577,577,577,577,577,577,577,577,577,577, -577,577,577,577,577,577,577,577,577,577,577,577,577,577,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, -119,119,119,119,119,119,119,119,119,119,119,119,119,119,119,119, - -/* block 263 */ -502, 24,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, -927,927,927,927,927,927,927,927,927,927,927,927,927,927,927,927, - -/* block 264 */ -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, /* block 265 */ -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21,970,970,970,970,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970, + 21, 21, 21, 21, 21,970,970,970, 21, 21, 21,970,970,970,970,970, /* block 266 */ -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -112,112,112,112,112,112,112,112,112,112,112,112,112,112,112,112, -502,502,502,502,502,502,502,502,502,502,502,502,502,502,502,502, + 21, 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970,970, + 21, 21, 21,970,970,970,970,970,970,970,970,970,970,970,970,970, + 21, 21, 21, 21, 21, 21, 21,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, /* block 267 */ -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,659,659, -659,659,659,659,659,659,659,659,659,659,659,659,659,659,119,119, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20,120, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,120,120,120,120,120,120, + +/* block 268 */ +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,970,970, +970,970,970,970,970,970,970,970,970,970,970,970,970,970,120,120, + +/* block 269 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 270 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,120,120,120,120,120,120,120,120,120,120,120, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, + +/* block 271 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,120,120, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, + +/* block 272 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, + +/* block 273 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 274 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 275 */ +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,586,586,586,586,586, +586,586,586,586,586,586,586,586,586,586,586,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, +120,120,120,120,120,120,120,120,120,120,120,120,120,120,120,120, + +/* block 276 */ +511, 24,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, +974,974,974,974,974,974,974,974,974,974,974,974,974,974,974,974, + +/* block 277 */ +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, + +/* block 278 */ +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, + +/* block 279 */ +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +113,113,113,113,113,113,113,113,113,113,113,113,113,113,113,113, +511,511,511,511,511,511,511,511,511,511,511,511,511,511,511,511, + +/* block 280 */ +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,673,673, +673,673,673,673,673,673,673,673,673,673,673,673,673,673,120,120, }; diff --git a/dist2/src/pcre2_ucp.h b/dist2/src/pcre2_ucp.h index 483abd18..9538062c 100644 --- a/dist2/src/pcre2_ucp.h +++ b/dist2/src/pcre2_ucp.h @@ -281,7 +281,17 @@ enum { ucp_Makasar, ucp_Medefaidrin, ucp_Old_Sogdian, - ucp_Sogdian + ucp_Sogdian, + /* New for Unicode 12.0.0 */ + ucp_Elymaic, + ucp_Nandinagari, + ucp_Nyiakeng_Puachue_Hmong, + ucp_Wancho, + /* New for Unicode 13.0.0 */ + ucp_Chorasmian, + ucp_Dives_Akuru, + ucp_Khitan_Small_Script, + ucp_Yezidi }; #endif /* PCRE2_UCP_H_IDEMPOTENT_GUARD */ diff --git a/dist2/src/pcre2_valid_utf.c b/dist2/src/pcre2_valid_utf.c index 96e8bff9..e47ea78f 100644 --- a/dist2/src/pcre2_valid_utf.c +++ b/dist2/src/pcre2_valid_utf.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2017 University of Cambridge + New API code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -347,7 +347,7 @@ for (p = string; length > 0; p++) length--; if ((*p & 0xfc00) != 0xdc00) { - *erroroffset = p - string; + *erroroffset = p - string - 1; return PCRE2_ERROR_UTF16_ERR2; } } diff --git a/dist2/src/pcre2demo.c b/dist2/src/pcre2demo.c index 5d9b3217..a49f1f8e 100644 --- a/dist2/src/pcre2demo.c +++ b/dist2/src/pcre2demo.c @@ -87,12 +87,11 @@ uint32_t newline; PCRE2_SIZE erroroffset; PCRE2_SIZE *ovector; +PCRE2_SIZE subject_length; -size_t subject_length; pcre2_match_data *match_data; - /************************************************************************** * First, sort out the command line. There is only one possible option at * * the moment, "-g" to request repeated matching to find all occurrences, * @@ -121,12 +120,14 @@ if (argc - i != 2) return 1; } -/* As pattern and subject are char arguments, they can be straightforwardly -cast to PCRE2_SPTR as we are working in 8-bit code units. */ +/* Pattern and subject are char arguments, so they can be straightforwardly +cast to PCRE2_SPTR because we are working in 8-bit code units. The subject +length is cast to PCRE2_SIZE for completeness, though PCRE2_SIZE is in fact +defined to be size_t. */ pattern = (PCRE2_SPTR)argv[i]; subject = (PCRE2_SPTR)argv[i+1]; -subject_length = strlen((char *)subject); +subject_length = (PCRE2_SIZE)strlen((char *)subject); /************************************************************************* @@ -155,17 +156,22 @@ if (re == NULL) /************************************************************************* -* If the compilation succeeded, we call PCRE again, in order to do a * +* If the compilation succeeded, we call PCRE2 again, in order to do a * * pattern match against the subject string. This does just ONE match. If * * further matching is needed, it will be done below. Before running the * -* match we must set up a match_data block for holding the result. * +* match we must set up a match_data block for holding the result. Using * +* pcre2_match_data_create_from_pattern() ensures that the block is * +* exactly the right size for the number of capturing parentheses in the * +* pattern. If you need to know the actual size of a match_data block as * +* a number of bytes, you can find it like this: * +* * +* PCRE2_SIZE match_data_size = pcre2_get_match_data_size(match_data); * *************************************************************************/ -/* Using this function ensures that the block is exactly the right size for -the number of capturing parentheses in the pattern. */ - match_data = pcre2_match_data_create_from_pattern(re, NULL); +/* Now run the match. */ + rc = pcre2_match( re, /* the compiled pattern */ subject, /* the subject string */ @@ -188,7 +194,7 @@ if (rc < 0) default: printf("Matching error %d\n", rc); break; } pcre2_match_data_free(match_data); /* Release memory used for the match */ - pcre2_code_free(re); /* data and the compiled pattern. */ + pcre2_code_free(re); /* data and the compiled pattern. */ return 1; } @@ -232,7 +238,7 @@ application you might want to do things other than print them. */ for (i = 0; i < rc; i++) { PCRE2_SPTR substring_start = subject + ovector[2*i]; - size_t substring_length = ovector[2*i+1] - ovector[2*i]; + PCRE2_SIZE substring_length = ovector[2*i+1] - ovector[2*i]; printf("%2d: %.*s\n", i, (int)substring_length, (char *)substring_start); } diff --git a/dist2/src/pcre2grep.c b/dist2/src/pcre2grep.c index a3cc3ec2..b54229b2 100644 --- a/dist2/src/pcre2grep.c +++ b/dist2/src/pcre2grep.c @@ -13,7 +13,7 @@ distribution because other apparatus is needed to compile pcre2grep for z/OS. The header can be found in the special z/OS distribution, which is available from www.zaconsultants.net or from www.cbttape.org. - Copyright (c) 1997-2018 University of Cambridge + Copyright (c) 1997-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -110,12 +110,25 @@ MSVC 10/2010. Except for VC6 (which is missing some fundamentals and fails). */ #define snprintf _snprintf #endif +/* VC and older compilers don't support %td or %zu, and even some that claim to +be C99 don't support it (hence DISABLE_PERCENT_ZT). */ + +#if defined(_MSC_VER) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L || defined(DISABLE_PERCENT_ZT) +#define PTR_FORM "lu" +#define SIZ_FORM "lu" +#define SIZ_CAST (unsigned long int) +#else +#define PTR_FORM "td" +#define SIZ_FORM "zu" +#define SIZ_CAST +#endif + #define FALSE 0 #define TRUE 1 typedef int BOOL; -#define OFFSET_SIZE 33 +#define DEFAULT_CAPTURE_MAX 50 #if BUFSIZ > 8192 #define MAXPATLEN BUFSIZ @@ -151,6 +164,10 @@ enum { DEE_READ, DEE_SKIP }; enum { BIN_BINARY, BIN_NOMATCH, BIN_TEXT }; +/* Return values from decode_dollar_escape() */ + +enum { DDE_ERROR, DDE_CAPTURE, DDE_CHAR }; + /* In newer versions of gcc, with FORTIFY_SOURCE set (the default in some environments), a warning is issued if the value of fwrite() is ignored. Unfortunately, casting to (void) does not suppress the warning. To get round @@ -166,13 +183,21 @@ handled by using STDOUT_NL as the newline string. We also use a normal double quote for the example, as single quotes aren't usually available. */ #ifdef WIN32 -#define STDOUT_NL "\r\n" -#define QUOT "\"" +#define STDOUT_NL "\r\n" +#define STDOUT_NL_LEN 2 +#define QUOT "\"" #else -#define STDOUT_NL "\n" -#define QUOT "'" +#define STDOUT_NL "\n" +#define STDOUT_NL_LEN 1 +#define QUOT "'" #endif +/* This code is returned from decode_dollar_escape() when $n is encountered, +and used to mean "output STDOUT_NL". It is, of course, not a valid Unicode code +point. */ + +#define STDOUT_NL_CODE 0x7fffffffu + /************************************************* @@ -211,8 +236,9 @@ static int max_bufthird = PCRE2GREP_MAX_BUFSIZE; static int bufsize = 3*PCRE2GREP_BUFSIZE; static int endlinetype; -static unsigned long int total_count = 0; +static int count_limit = -1; /* Not long, so that it works with OP_NUMBER */ static unsigned long int counts_printed = 0; +static unsigned long int total_count = 0; #ifdef WIN32 static int dee_action = dee_SKIP; @@ -242,6 +268,8 @@ static pcre2_compile_context *compile_context; static pcre2_match_context *match_context; static pcre2_match_data *match_data; static PCRE2_SIZE *offsets; +static uint32_t offset_size; +static uint32_t capture_max = DEFAULT_CAPTURE_MAX; static BOOL count_only = FALSE; static BOOL do_colour = FALSE; @@ -262,6 +290,9 @@ static BOOL show_total_count = FALSE; static BOOL silent = FALSE; static BOOL utf = FALSE; +static uint8_t utf8_buffer[8]; + + /* Structure for list of --only-matching capturing numbers. */ typedef struct omstr { @@ -391,6 +422,7 @@ used to identify them. */ #define N_INCLUDE_FROM (-21) #define N_OM_SEPARATOR (-22) #define N_MAX_BUFSIZE (-23) +#define N_OM_CAPTURE (-24) static option_item optionlist[] = { { OP_NODATA, N_NULL, NULL, "", "terminate options" }, @@ -427,6 +459,7 @@ static option_item optionlist[] = { { OP_U32NUMBER, N_M_LIMIT_DEP, &depth_limit, "depth-limit=number", "set PCRE2 depth limit option" }, { OP_U32NUMBER, N_M_LIMIT_DEP, &depth_limit, "recursion-limit=number", "obsolete synonym for depth-limit" }, { OP_NODATA, 'M', NULL, "multiline", "run in multiline mode" }, + { OP_NUMBER, 'm', &count_limit, "max-count=number", "stop after <number> matched lines" }, { OP_STRING, 'N', &newline_arg, "newline=type", "set newline type (CR, LF, CRLF, ANYCRLF, ANY, or NUL)" }, { OP_NODATA, 'n', NULL, "line-number", "print line number with output lines" }, #ifdef SUPPORT_PCRE2GREP_JIT @@ -437,6 +470,7 @@ static option_item optionlist[] = { { OP_STRING, 'O', &output_text, "output=text", "show only this text (possibly expanded)" }, { OP_OP_NUMBERS, 'o', &only_matching_data, "only-matching=n", "show only the part of the line that matched" }, { OP_STRING, N_OM_SEPARATOR, &om_separator, "om-separator=text", "set separator for multiple -o output" }, + { OP_U32NUMBER, N_OM_CAPTURE, &capture_max, "om-capture=n", "set capture count for --only-matching" }, { OP_NODATA, 'q', NULL, "quiet", "suppress output, just set return code" }, { OP_NODATA, 'r', NULL, "recursive", "recursively scan sub-directories" }, { OP_PATLIST, N_EXCLUDE,&exclude_patdata, "exclude=pattern","exclude matching files when recursing" }, @@ -451,6 +485,7 @@ static option_item optionlist[] = { { OP_NODATA, 's', NULL, "no-messages", "suppress error messages" }, { OP_NODATA, 't', NULL, "total-count", "print total count of matching lines" }, { OP_NODATA, 'u', NULL, "utf", "use UTF mode" }, + { OP_NODATA, 'U', NULL, "utf-allow-invalid", "use UTF mode, allow for invalid code units" }, { OP_NODATA, 'V', NULL, "version", "print version information and exit" }, { OP_NODATA, 'v', NULL, "invert-match", "select non-matching lines" }, { OP_NODATA, 'w', NULL, "word-regex(p)", "force patterns to match only as words" }, @@ -464,8 +499,13 @@ of PCRE2_NEWLINE_xx in pcre2.h. */ static const char *newlines[] = { "DEFAULT", "CR", "LF", "CRLF", "ANY", "ANYCRLF", "NUL" }; -/* UTF-8 tables - used only when the newline setting is "any". */ +/* UTF-8 tables */ + +const int utf8_table1[] = + { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff}; +const int utf8_table1_size = sizeof(utf8_table1) / sizeof(int); +const int utf8_table2[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; const char utf8_table4[] = { @@ -513,6 +553,32 @@ else #endif /* not VPCOMPAT && not HAVE_MEMMOVE */ + +/************************************************* +* Convert code point to UTF-8 * +*************************************************/ + +/* A static buffer is used. Returns the number of bytes. */ + +static int +ord2utf8(uint32_t value) +{ +int i, j; +uint8_t *utf8bytes = utf8_buffer; +for (i = 0; i < utf8_table1_size; i++) + if (value <= (uint32_t)utf8_table1[i]) break; +utf8bytes += i; +for (j = i; j > 0; j--) + { + *utf8bytes-- = 0x80 | (value & 0x3f); + value >>= 6; + } +*utf8bytes = utf8_table2[i] | value; +return i + 1; +} + + + /************************************************* * Case-independent string compare * *************************************************/ @@ -1648,6 +1714,44 @@ switch(endlinetype) /************************************************* +* Output newline at end * +*************************************************/ + +/* This function is called if the final line of a file has been written to +stdout, but it does not have a terminating newline. + +Arguments: none +Returns: nothing +*/ + +static void +write_final_newline(void) +{ +switch(endlinetype) + { + default: /* Just in case */ + case PCRE2_NEWLINE_LF: + case PCRE2_NEWLINE_ANY: + case PCRE2_NEWLINE_ANYCRLF: + fprintf(stdout, "\n"); + break; + + case PCRE2_NEWLINE_CR: + fprintf(stdout, "\r"); + break; + + case PCRE2_NEWLINE_CRLF: + fprintf(stdout, "\r\n"); + break; + + case PCRE2_NEWLINE_NUL: + fprintf(stdout, "%c", 0); + break; + } +} + + +/************************************************* * Print the previous "after" lines * *************************************************/ @@ -1671,9 +1775,9 @@ do_after_lines(unsigned long int lastmatchnumber, char *lastmatchrestart, if (after_context > 0 && lastmatchnumber > 0) { int count = 0; + int ellength = 0; while (lastmatchrestart < endptr && count < after_context) { - int ellength; char *pp = end_of_line(lastmatchrestart, endptr, &ellength); if (ellength == 0 && pp == main_buffer + bufsize) break; if (printname != NULL) fprintf(stdout, "%s-", printname); @@ -1682,7 +1786,17 @@ if (after_context > 0 && lastmatchnumber > 0) lastmatchrestart = pp; count++; } - if (count > 0) hyphenpending = TRUE; + + /* If we have printed any lines, arrange for a hyphen separator if anything + else follows. Also, if the last line is the final line in the file and it had + no newline, add one. */ + + if (count > 0) + { + hyphenpending = TRUE; + if (ellength == 0 && lastmatchrestart >= endptr) + write_final_newline(); + } } } @@ -1722,6 +1836,7 @@ if (slen > 200) slen = 200; msg = "text that starts:\n\n"; } + for (i = 1; p != NULL; p = p->next, i++) { *mrc = pcre2_match(p->compiled, (PCRE2_SPTR)matchptr, (int)length, @@ -1733,6 +1848,15 @@ for (i = 1; p != NULL; p = p->next, i++) fprintf(stderr, "%s", msg); FWRITE_IGNORE(matchptr, 1, slen, stderr); /* In case binary zero included */ fprintf(stderr, "\n\n"); + if (*mrc <= PCRE2_ERROR_UTF8_ERR1 && + *mrc >= PCRE2_ERROR_UTF8_ERR21) + { + unsigned char mbuffer[256]; + PCRE2_SIZE startchar = pcre2_get_startchar(match_data); + (void)pcre2_get_error_message(*mrc, mbuffer, sizeof(mbuffer)); + fprintf(stderr, "%s at offset %" SIZ_FORM "\n\n", mbuffer, + SIZ_CAST startchar); + } if (*mrc == PCRE2_ERROR_MATCHLIMIT || *mrc == PCRE2_ERROR_DEPTHLIMIT || *mrc == PCRE2_ERROR_HEAPLIMIT || *mrc == PCRE2_ERROR_JIT_STACKLIMIT) resource_error = TRUE; @@ -1748,140 +1872,254 @@ return FALSE; /* No match, no errors */ } + /************************************************* -* Check output text for errors * +* Decode dollar escape sequence * *************************************************/ -static BOOL -syntax_check_output_text(PCRE2_SPTR string, BOOL callout) +/* Called from various places to decode $ escapes in output strings. The escape +sequences are as follows: + +$<digits> or ${<digits>} returns a capture number. However, if callout is TRUE, +zero is never returned; '0' is substituted. + +$a returns bell. +$b returns backspace. +$e returns escape. +$f returns form feed. +$n returns newline. +$r returns carriage return. +$t returns tab. +$v returns vertical tab. +$o<digits> returns the character represented by the given octal + number; up to three digits are processed. +$o{<digits>} does the same, up to 7 digits, but gives an error for mode-invalid + code points. +$x<digits> returns the character represented by the given hexadecimal + number; up to two digits are processed. +$x{<digits} does the same, up to 6 digits, but gives an error for mode-invalid + code points. +Any other character is substituted by itself. E.g: $$ is replaced by a single +dollar. + +Arguments: + begin the start of the whole string + string points to the $ + callout TRUE if in a callout (inhibits error messages) + value where to return a value + last where to return pointer to the last used character + +Returns: DDE_ERROR after a syntax error + DDE_CAPTURE if *value is a capture number + DDE_CHAR if *value is a character code +*/ + +static int +decode_dollar_escape(PCRE2_SPTR begin, PCRE2_SPTR string, BOOL callout, + uint32_t *value, PCRE2_SPTR *last) { -PCRE2_SPTR begin = string; -for (; *string != 0; string++) +uint32_t c = 0; +int base = 10; +int dcount; +int rc = DDE_CHAR; +BOOL brace = FALSE; + +switch (*(++string)) { - if (*string == '$') + case 0: /* Syntax error: a character must be present after $. */ + if (!callout) + fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", + (int)(string - begin), "no character after $"); + *last = string; + return DDE_ERROR; + + case '{': + brace = TRUE; + string++; + if (!isdigit(*string)) /* Syntax error: a decimal number required. */ { - PCRE2_SIZE capture_id = 0; - BOOL brace = FALSE; + if (!callout) + fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", + (int)(string - begin), "decimal number expected"); + rc = DDE_ERROR; + break; + } + + /* Fall through */ + + /* The maximum capture number is 65535, so any number greater than that will + always be an unknown capture number. We just stop incrementing, in order to + avoid overflow. */ + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + do + { + if (c <= 65535) c = c * 10 + (*string - '0'); string++; + } + while (*string >= '0' && *string <= '9'); + string--; /* Point to last digit */ - /* Syntax error: a character must be present after $. */ - if (*string == 0) - { - if (!callout) - fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", - (int)(string - begin), "no character after $"); - return FALSE; - } + /* In a callout, capture number 0 is not available. No error can be given, + so just return the character '0'. */ - if (*string == '{') - { - /* Must be a decimal number in braces, e.g: {5} or {38} */ - string++; + if (callout && c == 0) + { + *value = '0'; + } + else + { + *value = c; + rc = DDE_CAPTURE; + } + break; - brace = TRUE; - } + /* Limit octal numbers to 3 digits without braces, or up to 7 with braces, + for valid Unicode code points. */ - if ((*string >= '1' && *string <= '9') || (!callout && *string == '0')) - { - do - { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); + case 'o': + base = 8; + string++; + if (*string == '{') + { + brace = TRUE; + string++; + dcount = 7; + } + else dcount = 3; + for (; dcount > 0; dcount--) + { + if (*string < '0' || *string > '7') break; + c = c * 8 + (*string++ - '0'); + } + *value = c; + string--; /* Point to last digit */ + break; - string++; - } - while (*string >= '0' && *string <= '9'); + /* Limit hex numbers to 2 digits without braces, or up to 6 with braces, + for valid Unicode code points. */ + + case 'x': + base = 16; + string++; + if (*string == '{') + { + brace = TRUE; + string++; + dcount = 6; + } + else dcount = 2; + for (; dcount > 0; dcount--) + { + if (!isxdigit(*string)) break; + if (*string >= '0' && *string <= '9') + c = c *16 + *string++ - '0'; + else + c = c * 16 + (*string++ | 0x20) - 'a' + 10; + } + *value = c; + string--; /* Point to last digit */ + break; - if (brace) + case 'a': *value = '\a'; break; + case 'b': *value = '\b'; break; +#ifndef EBCDIC + case 'e': *value = '\033'; break; +#else + case 'e': *value = '\047'; break; +#endif + case 'f': *value = '\f'; break; + case 'n': *value = STDOUT_NL_CODE; break; + case 'r': *value = '\r'; break; + case 't': *value = '\t'; break; + case 'v': *value = '\v'; break; + + default: *value = *string; break; + } + +if (brace) + { + c = string[1]; + if (c != '}') + { + rc = DDE_ERROR; + if (!callout) + { + if ((base == 8 && c >= '0' && c <= '7') || + (base == 16 && isxdigit(c))) { - /* Syntax error: closing brace is missing. */ - if (*string != '}') - { - if (!callout) - fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", - (int)(string - begin), "missing closing brace"); - return FALSE; - } + fprintf(stderr, "pcre2grep: Error in output text at offset %d: " + "too many %s digits\n", (int)(string - begin), + (base == 8)? "octal" : "hex"); } else { - /* To negate the effect of the for. */ - string--; - } - } - else if (brace) - { - /* Syntax error: a decimal number required. */ - if (!callout) fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", - (int)(string - begin), "decimal number expected"); - return FALSE; - } - else if (*string == 'o') - { - string++; - - if (*string < '0' || *string > '7') - { - /* Syntax error: an octal number required. */ - if (!callout) - fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", - (int)(string - begin), "octal number expected"); - return FALSE; + (int)(string - begin), "missing closing brace"); } } - else if (*string == 'x') - { - string++; + } + else string++; + } - if (!isxdigit((unsigned char)*string)) - { - /* Syntax error: a hexdecimal number required. */ - if (!callout) - fprintf(stderr, "pcre2grep: Error in output text at offset %d: %s\n", - (int)(string - begin), "hexadecimal number expected"); - return FALSE; - } - } +/* Check maximum code point values, but take note of STDOUT_NL_CODE. */ + +if (rc == DDE_CHAR && *value != STDOUT_NL_CODE) + { + uint32_t max = utf? 0x0010ffffu : 0xffu; + if (*value > max) + { + if (!callout) + fprintf(stderr, "pcre2grep: Error in output text at offset %d: " + "code point greater than 0x%x is invalid\n", (int)(string - begin), max); + rc = DDE_ERROR; } } - return TRUE; +*last = string; +return rc; } + /************************************************* -* Display output text * +* Check output text for errors * *************************************************/ -/* Display the output text, which is assumed to have already been syntax -checked. Output may contain escape sequences started by the dollar sign. The -escape sequences are substituted as follows: +/* Called early, to get errors before doing anything for -O text; also called +from callouts to check before outputting. - $<digits> or ${<digits>} is replaced by the captured substring of the given - decimal number; zero will substitute the whole match. If the number is - greater than the number of capturing substrings, or if the capture is unset, - the replacement is empty. - - $a is replaced by bell. - $b is replaced by backspace. - $e is replaced by escape. - $f is replaced by form feed. - $n is replaced by newline. - $r is replaced by carriage return. - $t is replaced by tab. - $v is replaced by vertical tab. - - $o<digits> is replaced by the character represented by the given octal - number; up to three digits are processed. +Arguments: + string an --output text string + callout TRUE if in a callout (stops printing errors) - $x<digits> is replaced by the character represented by the given hexadecimal - number; up to two digits are processed. +Returns: TRUE if OK, FALSE on error +*/ - Any other character is substituted by itself. E.g: $$ is replaced by a single - dollar. +static BOOL +syntax_check_output_text(PCRE2_SPTR string, BOOL callout) +{ +uint32_t value; +PCRE2_SPTR begin = string; + +for (; *string != 0; string++) + { + if (*string == '$' && + decode_dollar_escape(begin, string, callout, &value, &string) == DDE_ERROR) + return FALSE; + } + +return TRUE; +} + + +/************************************************* +* Display output text * +*************************************************/ + +/* Display the output text, which is assumed to have already been syntax +checked. Output may contain escape sequences started by the dollar sign. Arguments: string: the output text @@ -1898,121 +2136,54 @@ static BOOL display_output_text(PCRE2_SPTR string, BOOL callout, PCRE2_SPTR subject, PCRE2_SIZE *ovector, PCRE2_SIZE capture_top) { +uint32_t value; BOOL printed = FALSE; +PCRE2_SPTR begin = string; for (; *string != 0; string++) { - int ch = EOF; if (*string == '$') { - PCRE2_SIZE capture_id = 0; - BOOL brace = FALSE; - - string++; - - if (*string == '{') - { - /* Must be a decimal number in braces, e.g: {5} or {38} */ - string++; - - brace = TRUE; - } - - if ((*string >= '1' && *string <= '9') || (!callout && *string == '0')) + switch(decode_dollar_escape(begin, string, callout, &value, &string)) { - do + case DDE_CHAR: + if (value == STDOUT_NL_CODE) { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); - - string++; + fprintf(stdout, STDOUT_NL); + printed = FALSE; + continue; } - while (*string >= '0' && *string <= '9'); + break; /* Will print value */ - if (!brace) - { - /* To negate the effect of the for. */ - string--; - } - - if (capture_id < capture_top) + case DDE_CAPTURE: + if (value < capture_top) { PCRE2_SIZE capturesize; - capture_id *= 2; - - capturesize = ovector[capture_id + 1] - ovector[capture_id]; + value *= 2; + capturesize = ovector[value + 1] - ovector[value]; if (capturesize > 0) { - print_match(subject + ovector[capture_id], capturesize); + print_match(subject + ovector[value], capturesize); printed = TRUE; } } - } - else if (*string == 'a') ch = '\a'; - else if (*string == 'b') ch = '\b'; -#ifndef EBCDIC - else if (*string == 'e') ch = '\033'; -#else - else if (*string == 'e') ch = '\047'; -#endif - else if (*string == 'f') ch = '\f'; - else if (*string == 'r') ch = '\r'; - else if (*string == 't') ch = '\t'; - else if (*string == 'v') ch = '\v'; - else if (*string == 'n') - { - fprintf(stdout, STDOUT_NL); - printed = FALSE; - } - else if (*string == 'o') - { - string++; - - ch = *string - '0'; - if (string[1] >= '0' && string[1] <= '7') - { - string++; - ch = ch * 8 + (*string - '0'); - } - if (string[1] >= '0' && string[1] <= '7') - { - string++; - ch = ch * 8 + (*string - '0'); - } - } - else if (*string == 'x') - { - string++; + continue; - if (*string >= '0' && *string <= '9') - ch = *string - '0'; - else - ch = (*string | 0x20) - 'a' + 10; - if (isxdigit((unsigned char)string[1])) - { - string++; - ch *= 16; - if (*string >= '0' && *string <= '9') - ch += *string - '0'; - else - ch += (*string | 0x20) - 'a' + 10; - } - } - else - { - ch = *string; + default: /* Should not occur */ + break; } } - else - { - ch = *string; - } - if (ch != EOF) + + else value = *string; /* Not a $ escape */ + + if (utf && value <= 127) fprintf(stdout, "%c", *string); else { - fprintf(stdout, "%c", ch); - printed = TRUE; + int i; + int n = ord2utf8(value); + for (i = 0; i < n; i++) fputc(utf8_buffer[i], stdout); } + + printed = TRUE; } return printed; @@ -2091,7 +2262,7 @@ int result = 0; (void)unused; /* Avoid compiler warning */ -/* Only callout with strings are supported. */ +/* Only callouts with strings are supported. */ if (string == NULL || length == 0) return 0; @@ -2110,83 +2281,43 @@ return 0; #else /* Checking syntax and compute the number of string fragments. Callout strings -are ignored in case of a syntax error. */ +are silently ignored in the event of a syntax error. */ while (length > 0) { if (*string == '|') { argsvectorlen++; - - /* Maximum 10000 arguments allowed. */ - if (argsvectorlen > 10000) return 0; + if (argsvectorlen > 10000) return 0; /* Too many args */ } + else if (*string == '$') { - PCRE2_SIZE capture_id = 0; - - string++; - length--; + uint32_t value; + PCRE2_SPTR begin = string; - /* Syntax error: a character must be present after $. */ - if (length == 0) return 0; - - if (*string >= '1' && *string <= '9') + switch (decode_dollar_escape(begin, string, TRUE, &value, &string)) { - do + case DDE_CAPTURE: + if (value < capture_top) { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); - - string++; - length--; + value *= 2; + argslen += ovector[value + 1] - ovector[value]; } - while (length > 0 && *string >= '0' && *string <= '9'); - - /* To negate the effect of string++ below. */ - string--; - length++; - } - else if (*string == '{') - { - /* Must be a decimal number in braces, e.g: {5} or {38} */ - string++; - length--; - - /* Syntax error: a decimal number required. */ - if (length == 0) return 0; - if (*string < '1' || *string > '9') return 0; - - do - { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); - - string++; - length--; + argslen--; /* Negate the effect of argslen++ below. */ + break; - /* Syntax error: no more characters */ - if (length == 0) return 0; - } - while (*string >= '0' && *string <= '9'); + case DDE_CHAR: + if (value == STDOUT_NL_CODE) argslen += STDOUT_NL_LEN - 1; + else if (utf && value > 127) argslen += ord2utf8(value) - 1; + break; - /* Syntax error: closing brace is missing. */ - if (*string != '}') return 0; + default: /* Should not occur */ + case DDE_ERROR: + return 0; } - if (capture_id > 0) - { - if (capture_id < capture_top) - { - capture_id *= 2; - argslen += ovector[capture_id + 1] - ovector[capture_id]; - } - - /* To negate the effect of argslen++ below. */ - argslen--; - } + length -= (string - begin); } string++; @@ -2194,6 +2325,8 @@ while (length > 0) argslen++; } +/* Get memory for the argument vector and its strings. */ + args = (char*)malloc(argslen); if (args == NULL) return 0; @@ -2204,9 +2337,10 @@ if (argsvector == NULL) return 0; } +/* Now reprocess the string and set up the arguments. */ + argsptr = args; argsvectorptr = argsvector; - *argsvectorptr++ = argsptr; length = calloutptr->callout_string_length; @@ -2219,68 +2353,56 @@ while (length > 0) *argsptr++ = '\0'; *argsvectorptr++ = argsptr; } + else if (*string == '$') { - string++; - length--; + uint32_t value; + PCRE2_SPTR begin = string; - if ((*string >= '1' && *string <= '9') || *string == '{') + switch (decode_dollar_escape(begin, string, TRUE, &value, &string)) { - PCRE2_SIZE capture_id = 0; - - if (*string != '{') + case DDE_CAPTURE: + if (value < capture_top) { - do - { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); - - string++; - length--; - } - while (length > 0 && *string >= '0' && *string <= '9'); + PCRE2_SIZE capturesize; + value *= 2; + capturesize = ovector[value + 1] - ovector[value]; + memcpy(argsptr, subject + ovector[value], capturesize); + argsptr += capturesize; + } + break; - /* To negate the effect of string++ below. */ - string--; - length++; + case DDE_CHAR: + if (value == STDOUT_NL_CODE) + { + memcpy(argsptr, STDOUT_NL, STDOUT_NL_LEN); + argsptr += STDOUT_NL_LEN; + } + else if (utf && value > 127) + { + int n = ord2utf8(value); + memcpy(argsptr, utf8_buffer, n); + argsptr += n; } else { - string++; - length--; - - do - { - /* Maximum capture id is 65535. */ - if (capture_id <= 65535) - capture_id = capture_id * 10 + (*string - '0'); - - string++; - length--; - } - while (*string != '}'); + *argsptr++ = value; } + break; - if (capture_id < capture_top) - { - PCRE2_SIZE capturesize; - capture_id *= 2; - - capturesize = ovector[capture_id + 1] - ovector[capture_id]; - memcpy(argsptr, subject + ovector[capture_id], capturesize); - argsptr += capturesize; - } - } - else - { - *argsptr++ = *string; + default: /* Even though this should not occur, the string having */ + case DDE_ERROR: /* been checked above, we need to include the free() */ + free(args); /* calls so that source checkers do not complain. */ + free(argsvector); + return 0; } + + length -= (string - begin); } - else - { - *argsptr++ = *string; - } + + else *argsptr++ = *string; + + /* Advance along the string */ string++; length--; @@ -2404,12 +2526,14 @@ int filepos = 0; unsigned long int linenumber = 1; unsigned long int lastmatchnumber = 0; unsigned long int count = 0; +long int count_matched_lines = 0; char *lastmatchrestart = main_buffer; char *ptr = main_buffer; char *endptr; PCRE2_SIZE bufflength; BOOL binary = FALSE; BOOL endhyphenpending = FALSE; +BOOL lines_printed = FALSE; BOOL input_line_buffered = line_buffered; FILE *in = NULL; /* Ensure initialized */ @@ -2429,7 +2553,7 @@ bufflength = fill_buffer(handle, frtype, main_buffer, bufsize, input_line_buffered); #ifdef SUPPORT_LIBBZ2 -if (frtype == FR_LIBBZ2 && (int)bufflength < 0) return 2; /* Gotcha: bufflength is PCRE2_SIZE; */ +if (frtype == FR_LIBBZ2 && (int)bufflength < 0) return 2; /* Gotcha: bufflength is PCRE2_SIZE */ #endif endptr = main_buffer + bufflength; @@ -2457,10 +2581,23 @@ while (ptr < endptr) int mrc = 0; unsigned int options = 0; BOOL match; + BOOL line_matched = FALSE; char *t = ptr; PCRE2_SIZE length, linelength; PCRE2_SIZE startoffset = 0; + /* If the -m option set a limit for the number of matched or non-matched + lines, check it here. A limit of zero means that no matching is ever done. + For stdin from a file, set the file position. */ + + if (count_limit >= 0 && count_matched_lines >= count_limit) + { + if (frtype == FR_PLAIN && filename == stdin_name && !is_file_tty(handle)) + (void)fseek(handle, (long int)filepos, SEEK_SET); + rc = (count_limit == 0)? 1 : 0; + break; + } + /* At this point, ptr is at the start of a line. We need to find the length of the subject string to pass to pcre2_match(). In multiline mode, it is the length remainder of the data in the buffer. Otherwise, it is the length of @@ -2568,7 +2705,7 @@ while (ptr < endptr) for (i = 0; i < jfriedl_XR; i++) match = (pcre_exec(patterns->compiled, patterns->hint, ptr, length, 0, - PCRE2_NOTEMPTY, offsets, OFFSET_SIZE) >= 0); + PCRE2_NOTEMPTY, offsets, offset_size) >= 0); if (gettimeofday(&end_time, &dummy) != 0) perror("bad gettimeofday"); @@ -2610,6 +2747,10 @@ while (ptr < endptr) if (filenames == FN_NOMATCH_ONLY) return 1; + /* Remember that this line matched (for counting matched lines) */ + + line_matched = TRUE; + /* If all we want is a yes/no answer, we can return immediately. */ if (quiet) return 0; @@ -2688,7 +2829,7 @@ while (ptr < endptr) for (om = only_matching; om != NULL; om = om->next) { int n = om->groupnum; - if (n < mrc) + if (n == 0 || n < mrc) { int plen = offsets[2*n + 1] - offsets[2*n]; if (plen > 0) @@ -2750,6 +2891,8 @@ while (ptr < endptr) else { + lines_printed = TRUE; + /* See if there is a requirement to print some "after" lines from a previous match. We never print any overlaps. */ @@ -2798,7 +2941,8 @@ while (ptr < endptr) int linecount = 0; char *p = ptr; - while (p > main_buffer && (lastmatchnumber == 0 || p > lastmatchrestart) && + while (p > main_buffer && + (lastmatchnumber == 0 || p > lastmatchrestart) && linecount < before_context) { linecount++; @@ -2954,6 +3098,12 @@ while (ptr < endptr) lastmatchrestart = ptr + linelength + endlinelength; lastmatchnumber = linenumber + 1; + + /* If a line was printed and we are now at the end of the file and the last + line had no newline, output one. */ + + if (lines_printed && lastmatchrestart >= endptr && endlinelength == 0) + write_final_newline(); } /* For a match in multiline inverted mode (which of course did not cause @@ -2982,6 +3132,11 @@ while (ptr < endptr) filepos += (int)(linelength + endlinelength); linenumber++; + /* If there was at least one match (or a non-match, as required) in the line, + increment the count for the -m option. */ + + if (line_matched) count_matched_lines++; + /* If input is line buffered, and the buffer is not yet full, read another line and add it into the buffer. */ @@ -3401,6 +3556,7 @@ switch(letter) case 's': silent = TRUE; break; case 't': show_total_count = TRUE; break; case 'u': options |= PCRE2_UTF; utf = TRUE; break; + case 'U': options |= PCRE2_UTF|PCRE2_MATCH_INVALID_UTF; utf = TRUE; break; case 'v': invert = TRUE; break; case 'w': extra_options |= PCRE2_EXTRA_MATCH_WORD; break; case 'x': extra_options |= PCRE2_EXTRA_MATCH_LINE; break; @@ -3639,6 +3795,7 @@ int rc = 1; BOOL only_one_at_top; patstr *cp; fnstr *fn; +omstr *om; const char *locale_from = "--locale"; #ifdef SUPPORT_PCRE2GREP_JIT @@ -3655,20 +3812,6 @@ must use STDOUT_NL to terminate lines. */ _setmode(_fileno(stdout), _O_BINARY); #endif -/* Set up a default compile and match contexts and a match data block. */ - -compile_context = pcre2_compile_context_create(NULL); -match_context = pcre2_match_context_create(NULL); -match_data = pcre2_match_data_create(OFFSET_SIZE, NULL); -offsets = pcre2_get_ovector_pointer(match_data); - -/* If string (script) callouts are supported, set up the callout processing -function. */ - -#ifdef SUPPORT_PCRE2GREP_CALLOUT -pcre2_set_callout(match_context, pcre2grep_callout, NULL); -#endif - /* Process the options */ for (i = 1; i < argc; i++) @@ -4015,12 +4158,41 @@ if (only_matching_count > 1) pcre2grep_exit(usage(2)); } + +/* Check that there is a big enough ovector for all -o settings. */ + +for (om = only_matching; om != NULL; om = om->next) + { + int n = om->groupnum; + if (n > (int)capture_max) + { + fprintf(stderr, "pcre2grep: Requested group %d cannot be captured.\n", n); + fprintf(stderr, "pcre2grep: Use --om-capture to increase the size of the capture vector.\n"); + goto EXIT2; + } + } + /* Check the text supplied to --output for errors. */ if (output_text != NULL && !syntax_check_output_text((PCRE2_SPTR)output_text, FALSE)) goto EXIT2; +/* Set up default compile and match contexts and a match data block. */ + +offset_size = capture_max + 1; +compile_context = pcre2_compile_context_create(NULL); +match_context = pcre2_match_context_create(NULL); +match_data = pcre2_match_data_create(offset_size, NULL); +offsets = pcre2_get_ovector_pointer(match_data); + +/* If string (script) callouts are supported, set up the callout processing +function. */ + +#ifdef SUPPORT_PCRE2GREP_CALLOUT +pcre2_set_callout(match_context, pcre2grep_callout, NULL); +#endif + /* Put limits into the match data block. */ if (heap_limit != PCRE2_UNSET) pcre2_set_heap_limit(match_context, heap_limit); @@ -4347,7 +4519,7 @@ if (jit_stack != NULL) pcre2_jit_stack_free(jit_stack); #endif free(main_buffer); -free((void *)character_tables); +if (character_tables != NULL) pcre2_maketables_free(NULL, character_tables); pcre2_compile_context_free(compile_context); pcre2_match_context_free(match_context); diff --git a/dist2/src/pcre2posix.c b/dist2/src/pcre2posix.c index 34a8d809..b24620a4 100644 --- a/dist2/src/pcre2posix.c +++ b/dist2/src/pcre2posix.c @@ -323,6 +323,7 @@ if (preg->re_pcre2_code == NULL) PCRE2_INFO_CAPTURECOUNT, &re_nsub); preg->re_nsub = (size_t)re_nsub; preg->re_match_data = pcre2_match_data_create(re_nsub + 1, NULL); +preg->re_erroffset = (size_t)(-1); /* No meaning after successful compile */ if (preg->re_match_data == NULL) { @@ -356,8 +357,6 @@ if ((eflags & REG_NOTBOL) != 0) options |= PCRE2_NOTBOL; if ((eflags & REG_NOTEOL) != 0) options |= PCRE2_NOTEOL; if ((eflags & REG_NOTEMPTY) != 0) options |= PCRE2_NOTEMPTY; -((regex_t *)preg)->re_erroffset = (size_t)(-1); /* Only has meaning after compile */ - /* When REG_NOSUB was specified, or if no vector has been passed in which to put captured strings, ensure that nmatch is zero. This will stop any attempt to write to pmatch. */ diff --git a/dist2/src/pcre2test.c b/dist2/src/pcre2test.c index 40107728..aa007f8f 100644 --- a/dist2/src/pcre2test.c +++ b/dist2/src/pcre2test.c @@ -11,7 +11,7 @@ hacked-up (non-) design had also run out of steam. Written by Philip Hazel Original code Copyright (c) 1997-2012 University of Cambridge - Rewritten code Copyright (c) 2016-2019 University of Cambridge + Rewritten code Copyright (c) 2016-2020 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -170,7 +170,9 @@ commented out the original, but kept it around just in case. */ #endif /* VC and older compilers don't support %td or %zu, and even some that claim to -be C99 don't support it (hence DISABLE_PERCENT_ZT). */ +be C99 don't support it (hence DISABLE_PERCENT_ZT). There are some non-C99 +environments where %lu gives a warning with 32-bit pointers. As there doesn't +seem to be an easy way round this, just live with it (the cases are rare). */ #if defined(_MSC_VER) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L || defined(DISABLE_PERCENT_ZT) #define PTR_FORM "lu" @@ -212,6 +214,12 @@ be C99 don't support it (hence DISABLE_PERCENT_ZT). */ #define REPLACE_MODSIZE 100 /* Field for reading 8-bit replacement */ #define VERSION_SIZE 64 /* Size of buffer for the version strings */ +/* Default JIT compile options */ + +#define JIT_DEFAULT (PCRE2_JIT_COMPLETE|\ + PCRE2_JIT_PARTIAL_SOFT|\ + PCRE2_JIT_PARTIAL_HARD) + /* Make sure the buffer into which replacement strings are copied is big enough to hold them as 32-bit code units. */ @@ -381,12 +389,14 @@ typedef struct cmdstruct { int value; } cmdstruct; -enum { CMD_FORBID_UTF, CMD_LOAD, CMD_NEWLINE_DEFAULT, CMD_PATTERN, - CMD_PERLTEST, CMD_POP, CMD_POPCOPY, CMD_SAVE, CMD_SUBJECT, CMD_UNKNOWN }; +enum { CMD_FORBID_UTF, CMD_LOAD, CMD_LOADTABLES, CMD_NEWLINE_DEFAULT, + CMD_PATTERN, CMD_PERLTEST, CMD_POP, CMD_POPCOPY, CMD_SAVE, CMD_SUBJECT, + CMD_UNKNOWN }; static cmdstruct cmdlist[] = { { "forbid_utf", CMD_FORBID_UTF }, { "load", CMD_LOAD }, + { "loadtables", CMD_LOADTABLES }, { "newline_default", CMD_NEWLINE_DEFAULT }, { "pattern", CMD_PATTERN }, { "perltest", CMD_PERLTEST }, @@ -494,13 +504,16 @@ so many of them that they are split into two fields. */ #define CTL2_SUBSTITUTE_CALLOUT 0x00000001u #define CTL2_SUBSTITUTE_EXTENDED 0x00000002u -#define CTL2_SUBSTITUTE_OVERFLOW_LENGTH 0x00000004u -#define CTL2_SUBSTITUTE_UNKNOWN_UNSET 0x00000008u -#define CTL2_SUBSTITUTE_UNSET_EMPTY 0x00000010u -#define CTL2_SUBJECT_LITERAL 0x00000020u -#define CTL2_CALLOUT_NO_WHERE 0x00000040u -#define CTL2_CALLOUT_EXTRA 0x00000080u -#define CTL2_ALLVECTOR 0x00000100u +#define CTL2_SUBSTITUTE_LITERAL 0x00000004u +#define CTL2_SUBSTITUTE_MATCHED 0x00000008u +#define CTL2_SUBSTITUTE_OVERFLOW_LENGTH 0x00000010u +#define CTL2_SUBSTITUTE_REPLACEMENT_ONLY 0x00000020u +#define CTL2_SUBSTITUTE_UNKNOWN_UNSET 0x00000040u +#define CTL2_SUBSTITUTE_UNSET_EMPTY 0x00000080u +#define CTL2_SUBJECT_LITERAL 0x00000100u +#define CTL2_CALLOUT_NO_WHERE 0x00000200u +#define CTL2_CALLOUT_EXTRA 0x00000400u +#define CTL2_ALLVECTOR 0x00000800u #define CTL2_NL_SET 0x40000000u /* Informational */ #define CTL2_BSR_SET 0x80000000u /* Informational */ @@ -522,7 +535,10 @@ different things in the two cases. */ #define CTL2_ALLPD (CTL2_SUBSTITUTE_CALLOUT|\ CTL2_SUBSTITUTE_EXTENDED|\ + CTL2_SUBSTITUTE_LITERAL|\ + CTL2_SUBSTITUTE_MATCHED|\ CTL2_SUBSTITUTE_OVERFLOW_LENGTH|\ + CTL2_SUBSTITUTE_REPLACEMENT_ONLY|\ CTL2_SUBSTITUTE_UNKNOWN_UNSET|\ CTL2_SUBSTITUTE_UNSET_EMPTY|\ CTL2_ALLVECTOR) @@ -602,126 +618,130 @@ typedef struct modstruct { } modstruct; static modstruct modlist[] = { - { "aftertext", MOD_PNDP, MOD_CTL, CTL_AFTERTEXT, PO(control) }, - { "allaftertext", MOD_PNDP, MOD_CTL, CTL_ALLAFTERTEXT, PO(control) }, - { "allcaptures", MOD_PND, MOD_CTL, CTL_ALLCAPTURES, PO(control) }, - { "allow_empty_class", MOD_PAT, MOD_OPT, PCRE2_ALLOW_EMPTY_CLASS, PO(options) }, - { "allow_surrogate_escapes", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES, CO(extra_options) }, - { "allusedtext", MOD_PNDP, MOD_CTL, CTL_ALLUSEDTEXT, PO(control) }, - { "allvector", MOD_PND, MOD_CTL, CTL2_ALLVECTOR, PO(control2) }, - { "alt_bsux", MOD_PAT, MOD_OPT, PCRE2_ALT_BSUX, PO(options) }, - { "alt_circumflex", MOD_PAT, MOD_OPT, PCRE2_ALT_CIRCUMFLEX, PO(options) }, - { "alt_verbnames", MOD_PAT, MOD_OPT, PCRE2_ALT_VERBNAMES, PO(options) }, - { "altglobal", MOD_PND, MOD_CTL, CTL_ALTGLOBAL, PO(control) }, - { "anchored", MOD_PD, MOD_OPT, PCRE2_ANCHORED, PD(options) }, - { "auto_callout", MOD_PAT, MOD_OPT, PCRE2_AUTO_CALLOUT, PO(options) }, - { "bad_escape_is_literal", MOD_CTC, MOD_OPT, PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL, CO(extra_options) }, - { "bincode", MOD_PAT, MOD_CTL, CTL_BINCODE, PO(control) }, - { "bsr", MOD_CTC, MOD_BSR, 0, CO(bsr_convention) }, - { "callout_capture", MOD_DAT, MOD_CTL, CTL_CALLOUT_CAPTURE, DO(control) }, - { "callout_data", MOD_DAT, MOD_INS, 0, DO(callout_data) }, - { "callout_error", MOD_DAT, MOD_IN2, 0, DO(cerror) }, - { "callout_extra", MOD_DAT, MOD_CTL, CTL2_CALLOUT_EXTRA, DO(control2) }, - { "callout_fail", MOD_DAT, MOD_IN2, 0, DO(cfail) }, - { "callout_info", MOD_PAT, MOD_CTL, CTL_CALLOUT_INFO, PO(control) }, - { "callout_no_where", MOD_DAT, MOD_CTL, CTL2_CALLOUT_NO_WHERE, DO(control2) }, - { "callout_none", MOD_DAT, MOD_CTL, CTL_CALLOUT_NONE, DO(control) }, - { "caseless", MOD_PATP, MOD_OPT, PCRE2_CASELESS, PO(options) }, - { "convert", MOD_PAT, MOD_CON, 0, PO(convert_type) }, - { "convert_glob_escape", MOD_PAT, MOD_CHR, 0, PO(convert_glob_escape) }, - { "convert_glob_separator", MOD_PAT, MOD_CHR, 0, PO(convert_glob_separator) }, - { "convert_length", MOD_PAT, MOD_INT, 0, PO(convert_length) }, - { "copy", MOD_DAT, MOD_NN, DO(copy_numbers), DO(copy_names) }, - { "copy_matched_subject", MOD_DAT, MOD_OPT, PCRE2_COPY_MATCHED_SUBJECT, DO(options) }, - { "debug", MOD_PAT, MOD_CTL, CTL_DEBUG, PO(control) }, - { "depth_limit", MOD_CTM, MOD_INT, 0, MO(depth_limit) }, - { "dfa", MOD_DAT, MOD_CTL, CTL_DFA, DO(control) }, - { "dfa_restart", MOD_DAT, MOD_OPT, PCRE2_DFA_RESTART, DO(options) }, - { "dfa_shortest", MOD_DAT, MOD_OPT, PCRE2_DFA_SHORTEST, DO(options) }, - { "dollar_endonly", MOD_PAT, MOD_OPT, PCRE2_DOLLAR_ENDONLY, PO(options) }, - { "dotall", MOD_PATP, MOD_OPT, PCRE2_DOTALL, PO(options) }, - { "dupnames", MOD_PATP, MOD_OPT, PCRE2_DUPNAMES, PO(options) }, - { "endanchored", MOD_PD, MOD_OPT, PCRE2_ENDANCHORED, PD(options) }, - { "escaped_cr_is_lf", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ESCAPED_CR_IS_LF, CO(extra_options) }, - { "expand", MOD_PAT, MOD_CTL, CTL_EXPAND, PO(control) }, - { "extended", MOD_PATP, MOD_OPT, PCRE2_EXTENDED, PO(options) }, - { "extended_more", MOD_PATP, MOD_OPT, PCRE2_EXTENDED_MORE, PO(options) }, - { "extra_alt_bsux", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ALT_BSUX, CO(extra_options) }, - { "find_limits", MOD_DAT, MOD_CTL, CTL_FINDLIMITS, DO(control) }, - { "firstline", MOD_PAT, MOD_OPT, PCRE2_FIRSTLINE, PO(options) }, - { "framesize", MOD_PAT, MOD_CTL, CTL_FRAMESIZE, PO(control) }, - { "fullbincode", MOD_PAT, MOD_CTL, CTL_FULLBINCODE, PO(control) }, - { "get", MOD_DAT, MOD_NN, DO(get_numbers), DO(get_names) }, - { "getall", MOD_DAT, MOD_CTL, CTL_GETALL, DO(control) }, - { "global", MOD_PNDP, MOD_CTL, CTL_GLOBAL, PO(control) }, - { "heap_limit", MOD_CTM, MOD_INT, 0, MO(heap_limit) }, - { "hex", MOD_PAT, MOD_CTL, CTL_HEXPAT, PO(control) }, - { "info", MOD_PAT, MOD_CTL, CTL_INFO, PO(control) }, - { "jit", MOD_PAT, MOD_IND, 7, PO(jit) }, - { "jitfast", MOD_PAT, MOD_CTL, CTL_JITFAST, PO(control) }, - { "jitstack", MOD_PNDP, MOD_INT, 0, PO(jitstack) }, - { "jitverify", MOD_PAT, MOD_CTL, CTL_JITVERIFY, PO(control) }, - { "literal", MOD_PAT, MOD_OPT, PCRE2_LITERAL, PO(options) }, - { "locale", MOD_PAT, MOD_STR, LOCALESIZE, PO(locale) }, - { "mark", MOD_PNDP, MOD_CTL, CTL_MARK, PO(control) }, - { "match_limit", MOD_CTM, MOD_INT, 0, MO(match_limit) }, - { "match_line", MOD_CTC, MOD_OPT, PCRE2_EXTRA_MATCH_LINE, CO(extra_options) }, - { "match_unset_backref", MOD_PAT, MOD_OPT, PCRE2_MATCH_UNSET_BACKREF, PO(options) }, - { "match_word", MOD_CTC, MOD_OPT, PCRE2_EXTRA_MATCH_WORD, CO(extra_options) }, - { "max_pattern_length", MOD_CTC, MOD_SIZ, 0, CO(max_pattern_length) }, - { "memory", MOD_PD, MOD_CTL, CTL_MEMORY, PD(control) }, - { "multiline", MOD_PATP, MOD_OPT, PCRE2_MULTILINE, PO(options) }, - { "never_backslash_c", MOD_PAT, MOD_OPT, PCRE2_NEVER_BACKSLASH_C, PO(options) }, - { "never_ucp", MOD_PAT, MOD_OPT, PCRE2_NEVER_UCP, PO(options) }, - { "never_utf", MOD_PAT, MOD_OPT, PCRE2_NEVER_UTF, PO(options) }, - { "newline", MOD_CTC, MOD_NL, 0, CO(newline_convention) }, - { "no_auto_capture", MOD_PAT, MOD_OPT, PCRE2_NO_AUTO_CAPTURE, PO(options) }, - { "no_auto_possess", MOD_PATP, MOD_OPT, PCRE2_NO_AUTO_POSSESS, PO(options) }, - { "no_dotstar_anchor", MOD_PAT, MOD_OPT, PCRE2_NO_DOTSTAR_ANCHOR, PO(options) }, - { "no_jit", MOD_DAT, MOD_OPT, PCRE2_NO_JIT, DO(options) }, - { "no_start_optimize", MOD_PATP, MOD_OPT, PCRE2_NO_START_OPTIMIZE, PO(options) }, - { "no_utf_check", MOD_PD, MOD_OPT, PCRE2_NO_UTF_CHECK, PD(options) }, - { "notbol", MOD_DAT, MOD_OPT, PCRE2_NOTBOL, DO(options) }, - { "notempty", MOD_DAT, MOD_OPT, PCRE2_NOTEMPTY, DO(options) }, - { "notempty_atstart", MOD_DAT, MOD_OPT, PCRE2_NOTEMPTY_ATSTART, DO(options) }, - { "noteol", MOD_DAT, MOD_OPT, PCRE2_NOTEOL, DO(options) }, - { "null_context", MOD_PD, MOD_CTL, CTL_NULLCONTEXT, PO(control) }, - { "offset", MOD_DAT, MOD_INT, 0, DO(offset) }, - { "offset_limit", MOD_CTM, MOD_SIZ, 0, MO(offset_limit)}, - { "ovector", MOD_DAT, MOD_INT, 0, DO(oveccount) }, - { "parens_nest_limit", MOD_CTC, MOD_INT, 0, CO(parens_nest_limit) }, - { "partial_hard", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_HARD, DO(options) }, - { "partial_soft", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_SOFT, DO(options) }, - { "ph", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_HARD, DO(options) }, - { "posix", MOD_PAT, MOD_CTL, CTL_POSIX, PO(control) }, - { "posix_nosub", MOD_PAT, MOD_CTL, CTL_POSIX|CTL_POSIX_NOSUB, PO(control) }, - { "posix_startend", MOD_DAT, MOD_IN2, 0, DO(startend) }, - { "ps", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_SOFT, DO(options) }, - { "push", MOD_PAT, MOD_CTL, CTL_PUSH, PO(control) }, - { "pushcopy", MOD_PAT, MOD_CTL, CTL_PUSHCOPY, PO(control) }, - { "pushtablescopy", MOD_PAT, MOD_CTL, CTL_PUSHTABLESCOPY, PO(control) }, - { "recursion_limit", MOD_CTM, MOD_INT, 0, MO(depth_limit) }, /* Obsolete synonym */ - { "regerror_buffsize", MOD_PAT, MOD_INT, 0, PO(regerror_buffsize) }, - { "replace", MOD_PND, MOD_STR, REPLACE_MODSIZE, PO(replacement) }, - { "stackguard", MOD_PAT, MOD_INT, 0, PO(stackguard_test) }, - { "startchar", MOD_PND, MOD_CTL, CTL_STARTCHAR, PO(control) }, - { "startoffset", MOD_DAT, MOD_INT, 0, DO(offset) }, - { "subject_literal", MOD_PATP, MOD_CTL, CTL2_SUBJECT_LITERAL, PO(control2) }, - { "substitute_callout", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_CALLOUT, PO(control2) }, - { "substitute_extended", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_EXTENDED, PO(control2) }, - { "substitute_overflow_length", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_OVERFLOW_LENGTH, PO(control2) }, - { "substitute_skip", MOD_PND, MOD_INT, 0, PO(substitute_skip) }, - { "substitute_stop", MOD_PND, MOD_INT, 0, PO(substitute_stop) }, - { "substitute_unknown_unset", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_UNKNOWN_UNSET, PO(control2) }, - { "substitute_unset_empty", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_UNSET_EMPTY, PO(control2) }, - { "tables", MOD_PAT, MOD_INT, 0, PO(tables_id) }, - { "ucp", MOD_PATP, MOD_OPT, PCRE2_UCP, PO(options) }, - { "ungreedy", MOD_PAT, MOD_OPT, PCRE2_UNGREEDY, PO(options) }, - { "use_length", MOD_PAT, MOD_CTL, CTL_USE_LENGTH, PO(control) }, - { "use_offset_limit", MOD_PAT, MOD_OPT, PCRE2_USE_OFFSET_LIMIT, PO(options) }, - { "utf", MOD_PATP, MOD_OPT, PCRE2_UTF, PO(options) }, - { "utf8_input", MOD_PAT, MOD_CTL, CTL_UTF8_INPUT, PO(control) }, - { "zero_terminate", MOD_DAT, MOD_CTL, CTL_ZERO_TERMINATE, DO(control) } + { "aftertext", MOD_PNDP, MOD_CTL, CTL_AFTERTEXT, PO(control) }, + { "allaftertext", MOD_PNDP, MOD_CTL, CTL_ALLAFTERTEXT, PO(control) }, + { "allcaptures", MOD_PND, MOD_CTL, CTL_ALLCAPTURES, PO(control) }, + { "allow_empty_class", MOD_PAT, MOD_OPT, PCRE2_ALLOW_EMPTY_CLASS, PO(options) }, + { "allow_surrogate_escapes", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES, CO(extra_options) }, + { "allusedtext", MOD_PNDP, MOD_CTL, CTL_ALLUSEDTEXT, PO(control) }, + { "allvector", MOD_PND, MOD_CTL, CTL2_ALLVECTOR, PO(control2) }, + { "alt_bsux", MOD_PAT, MOD_OPT, PCRE2_ALT_BSUX, PO(options) }, + { "alt_circumflex", MOD_PAT, MOD_OPT, PCRE2_ALT_CIRCUMFLEX, PO(options) }, + { "alt_verbnames", MOD_PAT, MOD_OPT, PCRE2_ALT_VERBNAMES, PO(options) }, + { "altglobal", MOD_PND, MOD_CTL, CTL_ALTGLOBAL, PO(control) }, + { "anchored", MOD_PD, MOD_OPT, PCRE2_ANCHORED, PD(options) }, + { "auto_callout", MOD_PAT, MOD_OPT, PCRE2_AUTO_CALLOUT, PO(options) }, + { "bad_escape_is_literal", MOD_CTC, MOD_OPT, PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL, CO(extra_options) }, + { "bincode", MOD_PAT, MOD_CTL, CTL_BINCODE, PO(control) }, + { "bsr", MOD_CTC, MOD_BSR, 0, CO(bsr_convention) }, + { "callout_capture", MOD_DAT, MOD_CTL, CTL_CALLOUT_CAPTURE, DO(control) }, + { "callout_data", MOD_DAT, MOD_INS, 0, DO(callout_data) }, + { "callout_error", MOD_DAT, MOD_IN2, 0, DO(cerror) }, + { "callout_extra", MOD_DAT, MOD_CTL, CTL2_CALLOUT_EXTRA, DO(control2) }, + { "callout_fail", MOD_DAT, MOD_IN2, 0, DO(cfail) }, + { "callout_info", MOD_PAT, MOD_CTL, CTL_CALLOUT_INFO, PO(control) }, + { "callout_no_where", MOD_DAT, MOD_CTL, CTL2_CALLOUT_NO_WHERE, DO(control2) }, + { "callout_none", MOD_DAT, MOD_CTL, CTL_CALLOUT_NONE, DO(control) }, + { "caseless", MOD_PATP, MOD_OPT, PCRE2_CASELESS, PO(options) }, + { "convert", MOD_PAT, MOD_CON, 0, PO(convert_type) }, + { "convert_glob_escape", MOD_PAT, MOD_CHR, 0, PO(convert_glob_escape) }, + { "convert_glob_separator", MOD_PAT, MOD_CHR, 0, PO(convert_glob_separator) }, + { "convert_length", MOD_PAT, MOD_INT, 0, PO(convert_length) }, + { "copy", MOD_DAT, MOD_NN, DO(copy_numbers), DO(copy_names) }, + { "copy_matched_subject", MOD_DAT, MOD_OPT, PCRE2_COPY_MATCHED_SUBJECT, DO(options) }, + { "debug", MOD_PAT, MOD_CTL, CTL_DEBUG, PO(control) }, + { "depth_limit", MOD_CTM, MOD_INT, 0, MO(depth_limit) }, + { "dfa", MOD_DAT, MOD_CTL, CTL_DFA, DO(control) }, + { "dfa_restart", MOD_DAT, MOD_OPT, PCRE2_DFA_RESTART, DO(options) }, + { "dfa_shortest", MOD_DAT, MOD_OPT, PCRE2_DFA_SHORTEST, DO(options) }, + { "dollar_endonly", MOD_PAT, MOD_OPT, PCRE2_DOLLAR_ENDONLY, PO(options) }, + { "dotall", MOD_PATP, MOD_OPT, PCRE2_DOTALL, PO(options) }, + { "dupnames", MOD_PATP, MOD_OPT, PCRE2_DUPNAMES, PO(options) }, + { "endanchored", MOD_PD, MOD_OPT, PCRE2_ENDANCHORED, PD(options) }, + { "escaped_cr_is_lf", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ESCAPED_CR_IS_LF, CO(extra_options) }, + { "expand", MOD_PAT, MOD_CTL, CTL_EXPAND, PO(control) }, + { "extended", MOD_PATP, MOD_OPT, PCRE2_EXTENDED, PO(options) }, + { "extended_more", MOD_PATP, MOD_OPT, PCRE2_EXTENDED_MORE, PO(options) }, + { "extra_alt_bsux", MOD_CTC, MOD_OPT, PCRE2_EXTRA_ALT_BSUX, CO(extra_options) }, + { "find_limits", MOD_DAT, MOD_CTL, CTL_FINDLIMITS, DO(control) }, + { "firstline", MOD_PAT, MOD_OPT, PCRE2_FIRSTLINE, PO(options) }, + { "framesize", MOD_PAT, MOD_CTL, CTL_FRAMESIZE, PO(control) }, + { "fullbincode", MOD_PAT, MOD_CTL, CTL_FULLBINCODE, PO(control) }, + { "get", MOD_DAT, MOD_NN, DO(get_numbers), DO(get_names) }, + { "getall", MOD_DAT, MOD_CTL, CTL_GETALL, DO(control) }, + { "global", MOD_PNDP, MOD_CTL, CTL_GLOBAL, PO(control) }, + { "heap_limit", MOD_CTM, MOD_INT, 0, MO(heap_limit) }, + { "hex", MOD_PAT, MOD_CTL, CTL_HEXPAT, PO(control) }, + { "info", MOD_PAT, MOD_CTL, CTL_INFO, PO(control) }, + { "jit", MOD_PAT, MOD_IND, 7, PO(jit) }, + { "jitfast", MOD_PAT, MOD_CTL, CTL_JITFAST, PO(control) }, + { "jitstack", MOD_PNDP, MOD_INT, 0, PO(jitstack) }, + { "jitverify", MOD_PAT, MOD_CTL, CTL_JITVERIFY, PO(control) }, + { "literal", MOD_PAT, MOD_OPT, PCRE2_LITERAL, PO(options) }, + { "locale", MOD_PAT, MOD_STR, LOCALESIZE, PO(locale) }, + { "mark", MOD_PNDP, MOD_CTL, CTL_MARK, PO(control) }, + { "match_invalid_utf", MOD_PAT, MOD_OPT, PCRE2_MATCH_INVALID_UTF, PO(options) }, + { "match_limit", MOD_CTM, MOD_INT, 0, MO(match_limit) }, + { "match_line", MOD_CTC, MOD_OPT, PCRE2_EXTRA_MATCH_LINE, CO(extra_options) }, + { "match_unset_backref", MOD_PAT, MOD_OPT, PCRE2_MATCH_UNSET_BACKREF, PO(options) }, + { "match_word", MOD_CTC, MOD_OPT, PCRE2_EXTRA_MATCH_WORD, CO(extra_options) }, + { "max_pattern_length", MOD_CTC, MOD_SIZ, 0, CO(max_pattern_length) }, + { "memory", MOD_PD, MOD_CTL, CTL_MEMORY, PD(control) }, + { "multiline", MOD_PATP, MOD_OPT, PCRE2_MULTILINE, PO(options) }, + { "never_backslash_c", MOD_PAT, MOD_OPT, PCRE2_NEVER_BACKSLASH_C, PO(options) }, + { "never_ucp", MOD_PAT, MOD_OPT, PCRE2_NEVER_UCP, PO(options) }, + { "never_utf", MOD_PAT, MOD_OPT, PCRE2_NEVER_UTF, PO(options) }, + { "newline", MOD_CTC, MOD_NL, 0, CO(newline_convention) }, + { "no_auto_capture", MOD_PAT, MOD_OPT, PCRE2_NO_AUTO_CAPTURE, PO(options) }, + { "no_auto_possess", MOD_PATP, MOD_OPT, PCRE2_NO_AUTO_POSSESS, PO(options) }, + { "no_dotstar_anchor", MOD_PAT, MOD_OPT, PCRE2_NO_DOTSTAR_ANCHOR, PO(options) }, + { "no_jit", MOD_DAT, MOD_OPT, PCRE2_NO_JIT, DO(options) }, + { "no_start_optimize", MOD_PATP, MOD_OPT, PCRE2_NO_START_OPTIMIZE, PO(options) }, + { "no_utf_check", MOD_PD, MOD_OPT, PCRE2_NO_UTF_CHECK, PD(options) }, + { "notbol", MOD_DAT, MOD_OPT, PCRE2_NOTBOL, DO(options) }, + { "notempty", MOD_DAT, MOD_OPT, PCRE2_NOTEMPTY, DO(options) }, + { "notempty_atstart", MOD_DAT, MOD_OPT, PCRE2_NOTEMPTY_ATSTART, DO(options) }, + { "noteol", MOD_DAT, MOD_OPT, PCRE2_NOTEOL, DO(options) }, + { "null_context", MOD_PD, MOD_CTL, CTL_NULLCONTEXT, PO(control) }, + { "offset", MOD_DAT, MOD_INT, 0, DO(offset) }, + { "offset_limit", MOD_CTM, MOD_SIZ, 0, MO(offset_limit)}, + { "ovector", MOD_DAT, MOD_INT, 0, DO(oveccount) }, + { "parens_nest_limit", MOD_CTC, MOD_INT, 0, CO(parens_nest_limit) }, + { "partial_hard", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_HARD, DO(options) }, + { "partial_soft", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_SOFT, DO(options) }, + { "ph", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_HARD, DO(options) }, + { "posix", MOD_PAT, MOD_CTL, CTL_POSIX, PO(control) }, + { "posix_nosub", MOD_PAT, MOD_CTL, CTL_POSIX|CTL_POSIX_NOSUB, PO(control) }, + { "posix_startend", MOD_DAT, MOD_IN2, 0, DO(startend) }, + { "ps", MOD_DAT, MOD_OPT, PCRE2_PARTIAL_SOFT, DO(options) }, + { "push", MOD_PAT, MOD_CTL, CTL_PUSH, PO(control) }, + { "pushcopy", MOD_PAT, MOD_CTL, CTL_PUSHCOPY, PO(control) }, + { "pushtablescopy", MOD_PAT, MOD_CTL, CTL_PUSHTABLESCOPY, PO(control) }, + { "recursion_limit", MOD_CTM, MOD_INT, 0, MO(depth_limit) }, /* Obsolete synonym */ + { "regerror_buffsize", MOD_PAT, MOD_INT, 0, PO(regerror_buffsize) }, + { "replace", MOD_PND, MOD_STR, REPLACE_MODSIZE, PO(replacement) }, + { "stackguard", MOD_PAT, MOD_INT, 0, PO(stackguard_test) }, + { "startchar", MOD_PND, MOD_CTL, CTL_STARTCHAR, PO(control) }, + { "startoffset", MOD_DAT, MOD_INT, 0, DO(offset) }, + { "subject_literal", MOD_PATP, MOD_CTL, CTL2_SUBJECT_LITERAL, PO(control2) }, + { "substitute_callout", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_CALLOUT, PO(control2) }, + { "substitute_extended", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_EXTENDED, PO(control2) }, + { "substitute_literal", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_LITERAL, PO(control2) }, + { "substitute_matched", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_MATCHED, PO(control2) }, + { "substitute_overflow_length", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_OVERFLOW_LENGTH, PO(control2) }, + { "substitute_replacement_only", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_REPLACEMENT_ONLY, PO(control2) }, + { "substitute_skip", MOD_PND, MOD_INT, 0, PO(substitute_skip) }, + { "substitute_stop", MOD_PND, MOD_INT, 0, PO(substitute_stop) }, + { "substitute_unknown_unset", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_UNKNOWN_UNSET, PO(control2) }, + { "substitute_unset_empty", MOD_PND, MOD_CTL, CTL2_SUBSTITUTE_UNSET_EMPTY, PO(control2) }, + { "tables", MOD_PAT, MOD_INT, 0, PO(tables_id) }, + { "ucp", MOD_PATP, MOD_OPT, PCRE2_UCP, PO(options) }, + { "ungreedy", MOD_PAT, MOD_OPT, PCRE2_UNGREEDY, PO(options) }, + { "use_length", MOD_PAT, MOD_CTL, CTL_USE_LENGTH, PO(control) }, + { "use_offset_limit", MOD_PAT, MOD_OPT, PCRE2_USE_OFFSET_LIMIT, PO(options) }, + { "utf", MOD_PATP, MOD_OPT, PCRE2_UTF, PO(options) }, + { "utf8_input", MOD_PAT, MOD_CTL, CTL_UTF8_INPUT, PO(control) }, + { "zero_terminate", MOD_DAT, MOD_CTL, CTL_ZERO_TERMINATE, DO(control) } }; #define MODLISTCOUNT sizeof(modlist)/sizeof(modstruct) @@ -939,6 +959,8 @@ static int *dfa_workspace = NULL; static const uint8_t *locale_tables = NULL; static const uint8_t *use_tables = NULL; static uint8_t locale_name[32]; +static uint8_t *tables3 = NULL; /* For binary-loaded tables */ +static uint32_t loadtables_length = 0; /* We need buffers for building 16/32-bit strings; 8-bit strings don't need rebuilding, but set up the same naming scheme for use in macros. The "buffer" @@ -2958,15 +2980,15 @@ return (int)(pp - p); *************************************************/ /* Must handle UTF-8 strings in utf8 mode. Yields number of characters printed. -For printing *MARK strings, a negative length is given. If handed a NULL file, -just counts chars without printing (because pchar() does that). */ +For printing *MARK strings, a negative length is given, indicating that the +length is in the first code unit. If handed a NULL file, this function just +counts chars without printing (because pchar() does that). */ static int pchars8(PCRE2_SPTR8 p, int length, BOOL utf, FILE *f) { uint32_t c = 0; int yield = 0; - -if (length < 0) length = p[-1]; +if (length < 0) length = *p++; while (length-- > 0) { if (utf) @@ -2995,13 +3017,14 @@ return yield; *************************************************/ /* Must handle UTF-16 strings in utf mode. Yields number of characters printed. -For printing *MARK strings, a negative length is given. If handed a NULL file, -just counts chars without printing. */ +For printing *MARK strings, a negative length is given, indicating that the +length is in the first code unit. If handed a NULL file, just counts chars +without printing. */ static int pchars16(PCRE2_SPTR16 p, int length, BOOL utf, FILE *f) { int yield = 0; -if (length < 0) length = p[-1]; +if (length < 0) length = *p++; while (length-- > 0) { uint32_t c = *p++ & 0xffff; @@ -3029,15 +3052,15 @@ return yield; *************************************************/ /* Must handle UTF-32 strings in utf mode. Yields number of characters printed. -For printing *MARK strings, a negative length is given. If handed a NULL file, -just counts chars without printing. */ +For printing *MARK strings, a negative length is given, indicating that the +length is in the first code unit. If handed a NULL file, just counts chars +without printing. */ static int pchars32(PCRE2_SPTR32 p, int length, BOOL utf, FILE *f) { int yield = 0; (void)(utf); /* Avoid compiler warning */ - -if (length < 0) length = p[-1]; +if (length < 0) length = *p++; while (length-- > 0) { uint32_t c = *p++; @@ -3262,6 +3285,11 @@ return 0; +/* This function is no longer used. Keep it around for a while, just in case it +needs to be re-instated. */ + +#ifdef NEVERNEVERNEVER + /************************************************* * Move back by so many characters * *************************************************/ @@ -3306,6 +3334,7 @@ else /* 16-bit mode */ return pp - (PCRE2_SPTR16)subject; } } +#endif /* NEVERNEVERNEVER */ @@ -4070,7 +4099,7 @@ Returns: nothing static void show_controls(uint32_t controls, uint32_t controls2, const char *before) { -fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", +fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", before, ((controls & CTL_AFTERTEXT) != 0)? " aftertext" : "", ((controls & CTL_ALLAFTERTEXT) != 0)? " allaftertext" : "", @@ -4108,7 +4137,10 @@ fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s ((controls & CTL_STARTCHAR) != 0)? " startchar" : "", ((controls2 & CTL2_SUBSTITUTE_CALLOUT) != 0)? " substitute_callout" : "", ((controls2 & CTL2_SUBSTITUTE_EXTENDED) != 0)? " substitute_extended" : "", + ((controls2 & CTL2_SUBSTITUTE_LITERAL) != 0)? " substitute_literal" : "", + ((controls2 & CTL2_SUBSTITUTE_MATCHED) != 0)? " substitute_matched" : "", ((controls2 & CTL2_SUBSTITUTE_OVERFLOW_LENGTH) != 0)? " substitute_overflow_length" : "", + ((controls2 & CTL2_SUBSTITUTE_REPLACEMENT_ONLY) != 0)? " substitute_replacement_only" : "", ((controls2 & CTL2_SUBSTITUTE_UNKNOWN_UNSET) != 0)? " substitute_unknown_unset" : "", ((controls2 & CTL2_SUBSTITUTE_UNSET_EMPTY) != 0)? " substitute_unset_empty" : "", ((controls & CTL_USE_LENGTH) != 0)? " use_length" : "", @@ -4136,7 +4168,7 @@ static void show_compile_options(uint32_t options, const char *before, const char *after) { if (options == 0) fprintf(outfile, "%s <none>%s", before, after); -else fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", +else fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s", before, ((options & PCRE2_ALT_BSUX) != 0)? " alt_bsux" : "", ((options & PCRE2_ALT_CIRCUMFLEX) != 0)? " alt_circumflex" : "", @@ -4153,6 +4185,7 @@ else fprintf(outfile, "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s% ((options & PCRE2_EXTENDED_MORE) != 0)? " extended_more" : "", ((options & PCRE2_FIRSTLINE) != 0)? " firstline" : "", ((options & PCRE2_LITERAL) != 0)? " literal" : "", + ((options & PCRE2_MATCH_INVALID_UTF) != 0)? " match_invalid_utf" : "", ((options & PCRE2_MATCH_UNSET_BACKREF) != 0)? " match_unset_backref" : "", ((options & PCRE2_MULTILINE) != 0)? " multiline" : "", ((options & PCRE2_NEVER_BACKSLASH_C) != 0)? " never_backslash_c" : "", @@ -4696,7 +4729,8 @@ if ((pat_patctl.control & CTL_INFO) != 0) } } - fprintf(outfile, "Subject length lower bound = %d\n", minlength); + if ((FLD(compiled_code, overall_options) & PCRE2_NO_START_OPTIMIZE) == 0) + fprintf(outfile, "Subject length lower bound = %d\n", minlength); if (pat_patctl.jit != 0 && (pat_patctl.control & CTL_JITVERIFY) != 0) { @@ -4766,12 +4800,13 @@ Arguments: buffptr point after the #command mode open mode fptr points to the FILE variable + name name of # command Returns: PR_OK or PR_ABEND */ static int -open_file(uint8_t *buffptr, const char *mode, FILE **fptr) +open_file(uint8_t *buffptr, const char *mode, FILE **fptr, const char *name) { char *endf; char *filename = (char *)buffptr; @@ -4781,7 +4816,7 @@ while (endf > filename && isspace(endf[-1])) endf--; if (endf == filename) { - fprintf(outfile, "** File name expected after #save\n"); + fprintf(outfile, "** File name expected after %s\n", name); return PR_ABEND; } @@ -4867,7 +4902,7 @@ switch(cmd) case CMD_PATTERN: (void)decode_modifiers(argptr, CTX_DEFPAT, &def_patctl, NULL); if (def_patctl.jit == 0 && (def_patctl.control & CTL_JITVERIFY) != 0) - def_patctl.jit = 7; + def_patctl.jit = JIT_DEFAULT; break; /* Set default subject modifiers */ @@ -4947,7 +4982,7 @@ switch(cmd) return PR_OK; } - rc = open_file(argptr+1, BINARY_OUTPUT_MODE, &f); + rc = open_file(argptr+1, BINARY_OUTPUT_MODE, &f, "#save"); if (rc != PR_OK) return rc; PCRE2_SERIALIZE_ENCODE(rc, patstack, patstacknext, &serial, &serial_size, @@ -4986,7 +5021,7 @@ switch(cmd) /* Load a set of compiled patterns from a file onto the stack */ case CMD_LOAD: - rc = open_file(argptr+1, BINARY_INPUT_MODE, &f); + rc = open_file(argptr+1, BINARY_INPUT_MODE, &f, "#load"); if (rc != PR_OK) return rc; serial_size = 0; @@ -5038,6 +5073,32 @@ switch(cmd) free(serial); break; + + /* Load a set of binary tables into tables3. */ + + case CMD_LOADTABLES: + rc = open_file(argptr+1, BINARY_INPUT_MODE, &f, "#loadtables"); + if (rc != PR_OK) return rc; + + if (tables3 == NULL) + { + (void)PCRE2_CONFIG(PCRE2_CONFIG_TABLES_LENGTH, &loadtables_length); + tables3 = malloc(loadtables_length); + } + + if (tables3 == NULL) + { + fprintf(outfile, "** Failed: malloc failed for #loadtables\n"); + yield = PR_ABEND; + } + else if (fread(tables3, 1, loadtables_length, f) != loadtables_length) + { + fprintf(outfile, "** Wrong return from fread()\n"); + yield = PR_ABEND; + } + + fclose(f); + break; } return yield; @@ -5075,6 +5136,14 @@ PCRE2_SIZE patlen; PCRE2_SIZE valgrind_access_length; PCRE2_SIZE erroroffset; +/* The perltest.sh script supports only / as a delimiter. */ + +if (restrict_for_perl_test && delimiter != '/') + { + fprintf(outfile, "** The only allowed delimiter after #perltest is '/'\n"); + return PR_ABEND; + } + /* Initialize the context and pattern/data controls for this test from the defaults. */ @@ -5114,7 +5183,11 @@ patlen = p - buffer - 2; /* Look for modifiers and options after the final delimiter. */ if (!decode_modifiers(p, CTX_PAT, &pat_patctl, NULL)) return PR_SKIP; -utf = (pat_patctl.options & PCRE2_UTF) != 0; + +/* Note that the match_invalid_utf option also sets utf when passed to +pcre2_compile(). */ + +utf = (pat_patctl.options & (PCRE2_UTF|PCRE2_MATCH_INVALID_UTF)) != 0; /* The utf8_input modifier is not allowed in 8-bit mode, and is mutually exclusive with the utf modifier. */ @@ -5161,7 +5234,7 @@ specified. */ if (pat_patctl.jit == 0 && (pat_patctl.control & (CTL_JITVERIFY|CTL_JITFAST)) != 0) - pat_patctl.jit = 7; + pat_patctl.jit = JIT_DEFAULT; /* Now copy the pattern to pbuffer8 for use in 8-bit testing and for reflecting in callouts. Convert from hex if requested (literal strings in quotes may be @@ -5349,8 +5422,19 @@ else switch (pat_patctl.tables_id) case 0: use_tables = NULL; break; case 1: use_tables = tables1; break; case 2: use_tables = tables2; break; + + case 3: + if (tables3 == NULL) + { + fprintf(outfile, "** 'Tables = 3' is invalid: binary tables have not " + "been loaded\n"); + return PR_SKIP; + } + use_tables = tables3; + break; + default: - fprintf(outfile, "** 'Tables' must specify 0, 1, or 2.\n"); + fprintf(outfile, "** 'Tables' must specify 0, 1, 2, or 3.\n"); return PR_SKIP; } @@ -5744,6 +5828,7 @@ if (TEST(compiled_code, !=, NULL) && pat_patctl.jit != 0) { int i; clock_t time_taken = 0; + for (i = 0; i < timeit; i++) { clock_t start_time; @@ -5752,7 +5837,7 @@ if (TEST(compiled_code, !=, NULL) && pat_patctl.jit != 0) pat_patctl.options|use_forbid_utf, &errorcode, &erroroffset, use_pat_context); start_time = clock(); - PCRE2_JIT_COMPILE(jitrc,compiled_code, pat_patctl.jit); + PCRE2_JIT_COMPILE(jitrc, compiled_code, pat_patctl.jit); time_taken += clock() - start_time; } total_jit_compile_time += time_taken; @@ -6234,7 +6319,7 @@ if (cb->mark != last_callout_mark) else { fprintf(outfile, "Latest Mark: "); - PCHARSV(cb->mark, 0, -1, utf, outfile); + PCHARSV(cb->mark, -1, -1, utf, outfile); putc('\n', outfile); } last_callout_mark = cb->mark; @@ -7206,6 +7291,7 @@ if (dat_datctl.replacement[0] != 0) uint8_t rbuffer[REPLACE_BUFFSIZE]; uint8_t nbuffer[REPLACE_BUFFSIZE]; uint32_t xoptions; + uint32_t emoption; /* External match option */ PCRE2_SIZE j, rlen, nsize, erroroffset; BOOL badutf = FALSE; @@ -7230,12 +7316,30 @@ if (dat_datctl.replacement[0] != 0) if ((dat_datctl.control & CTL_ALTGLOBAL) != 0) fprintf(outfile, "** Altglobal is not supported with replace: ignored\n"); - xoptions = (((dat_datctl.control & CTL_GLOBAL) == 0)? 0 : + /* Check for a test that does substitution after an initial external match. + If this is set, we run the external match, but leave the interpretation of + its output to pcre2_substitute(). */ + + emoption = ((dat_datctl.control2 & CTL2_SUBSTITUTE_MATCHED) == 0)? 0 : + PCRE2_SUBSTITUTE_MATCHED; + + if (emoption != 0) + { + PCRE2_MATCH(rc, compiled_code, pp, arg_ulen, dat_datctl.offset, + dat_datctl.options, match_data, use_dat_context); + } + + xoptions = emoption | + (((dat_datctl.control & CTL_GLOBAL) == 0)? 0 : PCRE2_SUBSTITUTE_GLOBAL) | (((dat_datctl.control2 & CTL2_SUBSTITUTE_EXTENDED) == 0)? 0 : PCRE2_SUBSTITUTE_EXTENDED) | + (((dat_datctl.control2 & CTL2_SUBSTITUTE_LITERAL) == 0)? 0 : + PCRE2_SUBSTITUTE_LITERAL) | (((dat_datctl.control2 & CTL2_SUBSTITUTE_OVERFLOW_LENGTH) == 0)? 0 : PCRE2_SUBSTITUTE_OVERFLOW_LENGTH) | + (((dat_datctl.control2 & CTL2_SUBSTITUTE_REPLACEMENT_ONLY) == 0)? 0 : + PCRE2_SUBSTITUTE_REPLACEMENT_ONLY) | (((dat_datctl.control2 & CTL2_SUBSTITUTE_UNKNOWN_UNSET) == 0)? 0 : PCRE2_SUBSTITUTE_UNKNOWN_UNSET) | (((dat_datctl.control2 & CTL2_SUBSTITUTE_UNSET_EMPTY) == 0)? 0 : @@ -7736,7 +7840,7 @@ for (gmatched = 0;; gmatched++) TESTFLD(match_data, mark, !=, NULL)) { fprintf(outfile, "MK: "); - PCHARSV(CASTFLD(void *, match_data, mark), 0, -1, utf, outfile); + PCHARSV(CASTFLD(void *, match_data, mark), -1, -1, utf, outfile); fprintf(outfile, "\n"); } @@ -7747,28 +7851,35 @@ for (gmatched = 0;; gmatched++) } /* End of handling a successful match */ /* There was a partial match. The value of ovector[0] is the bumpalong point, - that is, startchar, not any \K point that might have been passed. */ + that is, startchar, not any \K point that might have been passed. When JIT is + not in use, "allusedtext" may be set, in which case we indicate the leftmost + consulted character. */ else if (capcount == PCRE2_ERROR_PARTIAL) { - PCRE2_SIZE poffset; + PCRE2_SIZE leftchar; int backlength; int rubriclength = 0; + if ((dat_datctl.control & CTL_ALLUSEDTEXT) != 0) + { + leftchar = FLD(match_data, leftchar); + } + else leftchar = ovector[0]; + fprintf(outfile, "Partial match"); if ((dat_datctl.control & CTL_MARK) != 0 && TESTFLD(match_data, mark, !=, NULL)) { fprintf(outfile, ", mark="); - PCHARS(rubriclength, CASTFLD(void *, match_data, mark), 0, -1, utf, + PCHARS(rubriclength, CASTFLD(void *, match_data, mark), -1, -1, utf, outfile); rubriclength += 7; } fprintf(outfile, ": "); rubriclength += 15; - poffset = backchars(pp, ovector[0], maxlookbehind, utf); - PCHARS(backlength, pp, poffset, ovector[0] - poffset, utf, outfile); + PCHARS(backlength, pp, leftchar, ovector[0] - leftchar, utf, outfile); PCHARSV(pp, ovector[0], ulen - ovector[0], utf, outfile); if ((pat_patctl.control & CTL_JITVERIFY) != 0 && jit_was_used) @@ -7860,7 +7971,7 @@ for (gmatched = 0;; gmatched++) TESTFLD(match_data, mark, !=, NULL)) { fprintf(outfile, ", mark = "); - PCHARSV(CASTFLD(void *, match_data, mark), 0, -1, utf, outfile); + PCHARSV(CASTFLD(void *, match_data, mark), -1, -1, utf, outfile); } if ((pat_patctl.control & CTL_JITVERIFY) != 0 && jit_was_used) fprintf(outfile, " (JIT)"); @@ -8051,7 +8162,7 @@ Returns: nothing static void print_newline_config(uint32_t optval, BOOL isc) { -if (!isc) printf(" Newline sequence is "); +if (!isc) printf(" Default newline sequence is "); if (optval < sizeof(newlines)/sizeof(char *)) printf("%s\n", newlines[optval]); else @@ -8107,6 +8218,7 @@ printf(" -error <n,m,..> show messages for error numbers, then exit\n"); printf(" -help show usage information\n"); printf(" -i set default pattern modifier 'info'\n"); printf(" -jit set default pattern modifier 'jit'\n"); +printf(" -jitfast set default pattern modifier 'jitfast'\n"); printf(" -jitverify set default pattern modifier 'jitverify'\n"); printf(" -LM list pattern and subject modifiers, then exit\n"); printf(" -q quiet: do not output PCRE2 version number at start\n"); @@ -8276,6 +8388,15 @@ printf(" Default heap limit = %d kibibytes\n", optval); printf(" Default match limit = %d\n", optval); (void)PCRE2_CONFIG(PCRE2_CONFIG_DEPTHLIMIT, &optval); printf(" Default depth limit = %d\n", optval); + +#if defined SUPPORT_LIBREADLINE +printf(" pcre2test has libreadline support\n"); +#elif defined SUPPORT_LIBEDIT +printf(" pcre2test has libedit support\n"); +#else +printf(" pcre2test has neither libreadline nor libedit support\n"); +#endif + return 0; } @@ -8612,13 +8733,15 @@ while (argc > 1 && argv[op][0] == '-' && argv[op][1] != 0) else if (strcmp(arg, "-d") == 0) def_patctl.control |= CTL_DEBUG; else if (strcmp(arg, "-dfa") == 0) def_datctl.control |= CTL_DFA; else if (strcmp(arg, "-i") == 0) def_patctl.control |= CTL_INFO; - else if (strcmp(arg, "-jit") == 0 || strcmp(arg, "-jitverify") == 0) + else if (strcmp(arg, "-jit") == 0 || strcmp(arg, "-jitverify") == 0 || + strcmp(arg, "-jitfast") == 0) { - if (arg[4] != 0) def_patctl.control |= CTL_JITVERIFY; - def_patctl.jit = 7; /* full & partial */ + if (arg[4] == 'v') def_patctl.control |= CTL_JITVERIFY; + else if (arg[4] == 'f') def_patctl.control |= CTL_JITFAST; + def_patctl.jit = JIT_DEFAULT; /* full & partial */ #ifndef SUPPORT_JIT fprintf(stderr, "** Warning: JIT support is not available: " - "-jit[verify] calls functions that do nothing.\n"); + "-jit[fast|verify] calls functions that do nothing.\n"); #endif } @@ -8631,6 +8754,11 @@ while (argc > 1 && argv[op][0] == '-' && argv[op][1] != 0) showtotaltimes = arg[1] == 'T'; if (argc > 2 && (uli = strtoul(argv[op+1], &endptr, 10), *endptr == 0)) { + if (uli == 0) + { + fprintf(stderr, "** Argument for %s must not be zero\n", arg); + exit(1); + } if (U32OVERFLOW(uli)) { fprintf(stderr, "** Argument for %s is too big\n", arg); @@ -8788,8 +8916,8 @@ least 128 code units, because it is used for retrieving error messages. */ } /* End of -error handling */ /* Initialize things that cannot be done until we know which test mode we are -running in. Exercise the general context copying function, which is not -otherwise used. */ +running in. Exercise the general context copying and match data size functions, +which are not otherwise used. */ code_unit_size = test_mode/8; max_oveccount = DEFAULT_OVECCOUNT; @@ -8811,7 +8939,9 @@ max_oveccount = DEFAULT_OVECCOUNT; (void)G(pcre2_set_compile_extra_options_,BITS)(G(pat_context,BITS), 0); \ (void)G(pcre2_set_max_pattern_length_,BITS)(G(pat_context,BITS), 0); \ (void)G(pcre2_set_offset_limit_,BITS)(G(dat_context,BITS), 0); \ - (void)G(pcre2_set_recursion_memory_management_,BITS)(G(dat_context,BITS), my_malloc, my_free, NULL) + (void)G(pcre2_set_recursion_memory_management_,BITS)(G(dat_context,BITS), my_malloc, my_free, NULL); \ + (void)G(pcre2_get_match_data_size_,BITS)(G(match_data,BITS)) + /* Call the appropriate functions for the current mode, and exercise some functions that are not otherwise called. */ @@ -9033,6 +9163,7 @@ free(dbuffer); free(pbuffer8); free(dfa_workspace); free((void *)locale_tables); +free(tables3); PCRE2_MATCH_DATA_FREE(match_data); SUB1(pcre2_code_free, compiled_code); diff --git a/dist2/src/sljit/sljitConfig.h b/dist2/src/sljit/sljitConfig.h index d54b5e6f..1c821d28 100644 --- a/dist2/src/sljit/sljitConfig.h +++ b/dist2/src/sljit/sljitConfig.h @@ -24,15 +24,19 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_CONFIG_H_ -#define _SLJIT_CONFIG_H_ +#ifndef SLJIT_CONFIG_H_ +#define SLJIT_CONFIG_H_ -/* --------------------------------------------------------------------- */ -/* Custom defines */ -/* --------------------------------------------------------------------- */ +#ifdef __cplusplus +extern "C" { +#endif -/* Put your custom defines here. This empty section will never change - which helps maintaining patches (with diff / patch utilities). */ +/* + This file contains the basic configuration options for the SLJIT compiler + and their default values. These options can be overridden in the + sljitConfigPre.h header file when SLJIT_HAVE_CONFIG_PRE is set to a + non-zero value. +*/ /* --------------------------------------------------------------------- */ /* Architecture */ @@ -50,7 +54,7 @@ /* #define SLJIT_CONFIG_MIPS_32 1 */ /* #define SLJIT_CONFIG_MIPS_64 1 */ /* #define SLJIT_CONFIG_SPARC_32 1 */ -/* #define SLJIT_CONFIG_TILEGX 1 */ +/* #define SLJIT_CONFIG_S390X 1 */ /* #define SLJIT_CONFIG_AUTO 1 */ /* #define SLJIT_CONFIG_UNSUPPORTED 1 */ @@ -59,18 +63,19 @@ /* Utilities */ /* --------------------------------------------------------------------- */ -/* Useful for thread-safe compiling of global functions. */ -#ifndef SLJIT_UTIL_GLOBAL_LOCK -/* Enabled by default */ -#define SLJIT_UTIL_GLOBAL_LOCK 1 -#endif - -/* Implements a stack like data structure (by using mmap / VirtualAlloc). */ +/* Implements a stack like data structure (by using mmap / VirtualAlloc */ +/* or a custom allocator). */ #ifndef SLJIT_UTIL_STACK /* Enabled by default */ #define SLJIT_UTIL_STACK 1 #endif +/* Uses user provided allocator to allocate the stack (see SLJIT_UTIL_STACK) */ +#ifndef SLJIT_UTIL_SIMPLE_STACK_ALLOCATION +/* Disabled by default */ +#define SLJIT_UTIL_SIMPLE_STACK_ALLOCATION 0 +#endif + /* Single threaded application. Does not require any locks. */ #ifndef SLJIT_SINGLE_THREADED /* Disabled by default. */ @@ -97,15 +102,31 @@ /* When SLJIT_PROT_EXECUTABLE_ALLOCATOR is enabled SLJIT uses an allocator which does not set writable and executable - permission flags at the same time. The trade-of is increased - memory consumption and disabled dynamic code modifications. */ + permission flags at the same time. + Instead, it creates a shared memory segment (usually backed by a file) + and maps it twice, with different permissions, depending on the use + case. + The trade-off is increased use of virtual memory, incompatibility with + fork(), and some possible additional security risks by the use of + publicly accessible files for the generated code. */ #ifndef SLJIT_PROT_EXECUTABLE_ALLOCATOR /* Disabled by default. */ #define SLJIT_PROT_EXECUTABLE_ALLOCATOR 0 #endif +/* When SLJIT_WX_EXECUTABLE_ALLOCATOR is enabled SLJIT uses an + allocator which does not set writable and executable permission + flags at the same time. + Instead, it creates a new independent map on each invocation and + switches permissions at the underlying pages as needed. + The trade-off is increased memory use and degraded performance. */ +#ifndef SLJIT_WX_EXECUTABLE_ALLOCATOR +/* Disabled by default. */ +#define SLJIT_WX_EXECUTABLE_ALLOCATOR 0 #endif +#endif /* !SLJIT_EXECUTABLE_ALLOCATOR */ + /* Force cdecl calling convention even if a better calling convention (e.g. fastcall) is supported by the C compiler. If this option is disabled (this is the default), functions @@ -144,4 +165,8 @@ /* For further configurations, see the beginning of sljitConfigInternal.h */ +#ifdef __cplusplus +} /* extern "C" */ #endif + +#endif /* SLJIT_CONFIG_H_ */ diff --git a/dist2/src/sljit/sljitConfigInternal.h b/dist2/src/sljit/sljitConfigInternal.h index ba60311e..eb1132db 100644 --- a/dist2/src/sljit/sljitConfigInternal.h +++ b/dist2/src/sljit/sljitConfigInternal.h @@ -24,8 +24,22 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_CONFIG_INTERNAL_H_ -#define _SLJIT_CONFIG_INTERNAL_H_ +#ifndef SLJIT_CONFIG_INTERNAL_H_ +#define SLJIT_CONFIG_INTERNAL_H_ + +#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \ + || (defined SLJIT_DEBUG && SLJIT_DEBUG && (!defined(SLJIT_ASSERT) || !defined(SLJIT_UNREACHABLE))) +#include <stdio.h> +#endif + +#if (defined SLJIT_DEBUG && SLJIT_DEBUG \ + && (!defined(SLJIT_ASSERT) || !defined(SLJIT_UNREACHABLE) || !defined(SLJIT_HALT_PROCESS))) +#include <stdlib.h> +#endif + +#ifdef __cplusplus +extern "C" { +#endif /* SLJIT defines the following architecture dependent types and macros: @@ -67,30 +81,13 @@ Other macros: SLJIT_FUNC : calling convention attribute for both calling JIT from C and C calling back from JIT - SLJIT_W(number) : defining 64 bit constants on 64 bit architectures (compiler independent helper) + SLJIT_W(number) : defining 64 bit constants on 64 bit architectures (platform independent helper) */ /*****************/ /* Sanity check. */ /*****************/ -#if !((defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \ - || (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) \ - || (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) \ - || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) \ - || (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) \ - || (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \ - || (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) \ - || (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) \ - || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) \ - || (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) \ - || (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) \ - || (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) \ - || (defined SLJIT_CONFIG_AUTO && SLJIT_CONFIG_AUTO) \ - || (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)) -#error "An architecture must be selected" -#endif - #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \ + (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) \ + (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) \ @@ -99,15 +96,36 @@ + (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \ + (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) \ + (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) \ - + (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) \ + (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) \ + (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) \ + (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) \ + + (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \ + (defined SLJIT_CONFIG_AUTO && SLJIT_CONFIG_AUTO) \ + (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) >= 2 #error "Multiple architectures are selected" #endif +#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \ + && !(defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) \ + && !(defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) \ + && !(defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) \ + && !(defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) \ + && !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \ + && !(defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) \ + && !(defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) \ + && !(defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) \ + && !(defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) \ + && !(defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) \ + && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \ + && !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) \ + && !(defined SLJIT_CONFIG_AUTO && SLJIT_CONFIG_AUTO) +#if defined SLJIT_CONFIG_AUTO && !SLJIT_CONFIG_AUTO +#error "An architecture must be selected" +#else /* SLJIT_CONFIG_AUTO */ +#define SLJIT_CONFIG_AUTO 1 +#endif /* !SLJIT_CONFIG_AUTO */ +#endif /* !SLJIT_CONFIG */ + /********************************************************/ /* Automatic CPU detection (requires compiler support). */ /********************************************************/ @@ -140,8 +158,6 @@ #define SLJIT_CONFIG_MIPS_64 1 #elif defined(__sparc__) || defined(__sparc) #define SLJIT_CONFIG_SPARC_32 1 -#elif defined(__tilegx__) -#define SLJIT_CONFIG_TILEGX 1 #else /* Unsupported architecture */ #define SLJIT_CONFIG_UNSUPPORTED 1 @@ -191,6 +207,22 @@ #define SLJIT_CONFIG_SPARC 1 #endif +/***********************************************************/ +/* Intel Control-flow Enforcement Technology (CET) spport. */ +/***********************************************************/ + +#ifdef SLJIT_CONFIG_X86 + +#if defined(__CET__) && !(defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) +#define SLJIT_CONFIG_X86_CET 1 +#endif + +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined(__GNUC__) +#include <x86intrin.h> +#endif + +#endif /* SLJIT_CONFIG_X86 */ + /**********************************/ /* External function definitions. */ /**********************************/ @@ -214,6 +246,10 @@ #define SLJIT_MEMCPY(dest, src, len) memcpy(dest, src, len) #endif +#ifndef SLJIT_MEMMOVE +#define SLJIT_MEMMOVE(dest, src, len) memmove(dest, src, len) +#endif + #ifndef SLJIT_ZEROMEM #define SLJIT_ZEROMEM(dest, len) memset(dest, 0, len) #endif @@ -261,6 +297,7 @@ /* Type of public API functions. */ /*********************************/ +#ifndef SLJIT_API_FUNC_ATTRIBUTE #if (defined SLJIT_CONFIG_STATIC && SLJIT_CONFIG_STATIC) /* Static ABI functions. For all-in-one programs. */ @@ -274,6 +311,7 @@ #else #define SLJIT_API_FUNC_ATTRIBUTE #endif /* (defined SLJIT_CONFIG_STATIC && SLJIT_CONFIG_STATIC) */ +#endif /* defined SLJIT_API_FUNC_ATTRIBUTE */ /****************************/ /* Instruction cache flush. */ @@ -283,7 +321,7 @@ #if __has_builtin(__builtin___clear_cache) #define SLJIT_CACHE_FLUSH(from, to) \ - __builtin___clear_cache((char*)from, (char*)to) + __builtin___clear_cache((char*)(from), (char*)(to)) #endif /* __has_builtin(__builtin___clear_cache) */ #endif /* (!defined SLJIT_CACHE_FLUSH && defined __has_builtin) */ @@ -314,7 +352,7 @@ #elif (defined(__GNUC__) && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) #define SLJIT_CACHE_FLUSH(from, to) \ - __builtin___clear_cache((char*)from, (char*)to) + __builtin___clear_cache((char*)(from), (char*)(to)) #elif defined __ANDROID__ @@ -373,7 +411,7 @@ typedef long int sljit_sw; && !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \ && !(defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) \ && !(defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) \ - && !(defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) + && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) #define SLJIT_32BIT_ARCHITECTURE 1 #define SLJIT_WORD_SHIFT 2 typedef unsigned int sljit_uw; @@ -415,10 +453,14 @@ typedef double sljit_f64; #if (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) #define SLJIT_W(w) (w##l) #elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) +#ifdef _WIN64 #define SLJIT_W(w) (w##ll) -#else +#else /* !windows */ +#define SLJIT_W(w) (w##l) +#endif /* windows */ +#else /* 32 bit */ #define SLJIT_W(w) (w) -#endif +#endif /* unknown */ #endif /* !SLJIT_W */ @@ -447,7 +489,27 @@ typedef double sljit_f64; #define SLJIT_BIG_ENDIAN 1 #endif -#elif (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) +#ifndef SLJIT_MIPS_REV + +/* Auto detecting mips revision. */ +#if (defined __mips_isa_rev) && (__mips_isa_rev >= 6) +#define SLJIT_MIPS_REV 6 +#elif (defined __mips_isa_rev && __mips_isa_rev >= 1) \ + || (defined __clang__ && defined _MIPS_ARCH_OCTEON) \ + || (defined __clang__ && defined _MIPS_ARCH_P5600) +/* clang either forgets to define (clang-7) __mips_isa_rev at all + * or sets it to zero (clang-8,-9) for -march=octeon (MIPS64 R2+) + * and -march=p5600 (MIPS32 R5). + * It also sets the __mips macro to 64 or 32 for -mipsN when N <= 5 + * (should be set to N exactly) so we cannot rely on this too. + */ +#define SLJIT_MIPS_REV 1 +#endif + +#endif /* !SLJIT_MIPS_REV */ + +#elif (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) \ + || (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) #define SLJIT_BIG_ENDIAN 1 @@ -474,7 +536,8 @@ typedef double sljit_f64; || (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) \ || (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \ || (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) \ - || (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) + || (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) \ + || (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) #define SLJIT_UNALIGNED 1 #endif @@ -492,17 +555,19 @@ typedef double sljit_f64; #ifndef SLJIT_FUNC -#if (defined SLJIT_USE_CDECL_CALLING_CONVENTION && SLJIT_USE_CDECL_CALLING_CONVENTION) +#if (defined SLJIT_USE_CDECL_CALLING_CONVENTION && SLJIT_USE_CDECL_CALLING_CONVENTION) \ + || !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -/* Force cdecl. */ #define SLJIT_FUNC -#elif (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) - -#if defined(__GNUC__) && !defined(__APPLE__) +#elif defined(__GNUC__) && !defined(__APPLE__) +#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) #define SLJIT_FUNC __attribute__ ((fastcall)) #define SLJIT_X86_32_FASTCALL 1 +#else +#define SLJIT_FUNC +#endif /* gcc >= 3.4 */ #elif defined(_MSC_VER) @@ -516,16 +581,10 @@ typedef double sljit_f64; #else /* Unknown compiler. */ -/* The cdecl attribute is the default. */ -#define SLJIT_FUNC - -#endif - -#else /* Non x86-32 architectures. */ - +/* The cdecl calling convention is usually the x86 default. */ #define SLJIT_FUNC -#endif /* SLJIT_CONFIG_X86_32 */ +#endif /* SLJIT_USE_CDECL_CALLING_CONVENTION */ #endif /* !SLJIT_FUNC */ @@ -556,8 +615,16 @@ determine the next executed instruction after return. */ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size); SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr); SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void); -#define SLJIT_MALLOC_EXEC(size) sljit_malloc_exec(size) -#define SLJIT_FREE_EXEC(ptr) sljit_free_exec(ptr) +#define SLJIT_BUILTIN_MALLOC_EXEC(size, exec_allocator_data) sljit_malloc_exec(size) +#define SLJIT_BUILTIN_FREE_EXEC(ptr, exec_allocator_data) sljit_free_exec(ptr) + +#ifndef SLJIT_MALLOC_EXEC +#define SLJIT_MALLOC_EXEC(size, exec_allocator_data) SLJIT_BUILTIN_MALLOC_EXEC((size), (exec_allocator_data)) +#endif /* SLJIT_MALLOC_EXEC */ + +#ifndef SLJIT_FREE_EXEC +#define SLJIT_FREE_EXEC(ptr, exec_allocator_data) SLJIT_BUILTIN_FREE_EXEC((ptr), (exec_allocator_data)) +#endif /* SLJIT_FREE_EXEC */ #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); @@ -566,7 +633,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); #define SLJIT_EXEC_OFFSET(ptr) 0 #endif -#endif +#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ /**********************************************/ /* Registers and locals offset determination. */ @@ -642,11 +709,32 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); #define SLJIT_LOCALS_OFFSET_BASE ((16 + 1 + 6 + 2 + 1) * sizeof(sljit_sw)) #endif -#elif (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) +#elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) -#define SLJIT_NUMBER_OF_REGISTERS 10 -#define SLJIT_NUMBER_OF_SAVED_REGISTERS 5 -#define SLJIT_LOCALS_OFFSET_BASE 0 +/* + * https://refspecs.linuxbase.org/ELF/zSeries/lzsabi0_zSeries.html#STACKFRAME + * + * 160 + * .. FR6 + * .. FR4 + * .. FR2 + * 128 FR0 + * 120 R15 (used for SP) + * 112 R14 + * 104 R13 + * 96 R12 + * .. + * 48 R6 + * .. + * 16 R2 + * 8 RESERVED + * 0 SP + */ +#define SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE 160 + +#define SLJIT_NUMBER_OF_REGISTERS 12 +#define SLJIT_NUMBER_OF_SAVED_REGISTERS 8 +#define SLJIT_LOCALS_OFFSET_BASE SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE #elif (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) @@ -675,24 +763,16 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); /* Debug and verbose related macros. */ /*************************************/ -#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) -#include <stdio.h> -#endif - #if (defined SLJIT_DEBUG && SLJIT_DEBUG) #if !defined(SLJIT_ASSERT) || !defined(SLJIT_UNREACHABLE) /* SLJIT_HALT_PROCESS must halt the process. */ #ifndef SLJIT_HALT_PROCESS -#include <stdlib.h> - #define SLJIT_HALT_PROCESS() \ abort(); #endif /* !SLJIT_HALT_PROCESS */ -#include <stdio.h> - #endif /* !SLJIT_ASSERT || !SLJIT_UNREACHABLE */ /* Feel free to redefine these two macros. */ @@ -738,4 +818,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); #endif /* !SLJIT_COMPILE_ASSERT */ +#ifdef __cplusplus +} /* extern "C" */ #endif + +#endif /* SLJIT_CONFIG_INTERNAL_H_ */ diff --git a/dist2/src/sljit/sljitExecAllocator.c b/dist2/src/sljit/sljitExecAllocator.c index 3b37a975..61a32f23 100644 --- a/dist2/src/sljit/sljitExecAllocator.c +++ b/dist2/src/sljit/sljitExecAllocator.c @@ -72,9 +72,8 @@ alloc_chunk / free_chunk : * allocate executable system memory chunks * the size is always divisible by CHUNK_SIZE - allocator_grab_lock / allocator_release_lock : - * make the allocator thread safe - * can be empty if the OS (or the application) does not support threading + SLJIT_ALLOCATOR_LOCK / SLJIT_ALLOCATOR_UNLOCK : + * provided as part of sljitUtils * only the allocator requires this lock, sljit is fully thread safe as it only uses local variables */ @@ -95,6 +94,7 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) #else #ifdef __APPLE__ +#ifdef MAP_ANON /* Configures TARGET_OS_OSX when appropriate */ #include <TargetConditionals.h> @@ -104,24 +104,41 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) #ifdef MAP_JIT +/* + On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a + version where it's OK to have more than one JIT block. + On non-macOS systems, returns MAP_JIT if it is defined. +*/ static SLJIT_INLINE int get_map_jit_flag() { #if TARGET_OS_OSX - /* On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a version - of macOS where it's OK to have more than one JIT block. On non-macOS systems, returns - MAP_JIT if it is defined. */ + sljit_sw page_size = get_page_alignment() + 1; + void *ptr; static int map_jit_flag = -1; - /* The following code is thread safe because multiple initialization - sets map_jit_flag to the same value and the code has no side-effects. - Changing the kernel version witout system restart is (very) unlikely. */ + /* + The following code is thread safe because multiple initialization + sets map_jit_flag to the same value and the code has no side-effects. + Changing the kernel version witout system restart is (very) unlikely. + */ if (map_jit_flag == -1) { struct utsname name; + map_jit_flag = 0; uname(&name); /* Kernel version for 10.14.0 (Mojave) */ - map_jit_flag = (atoi(name.release) >= 18) ? MAP_JIT : 0; + if (atoi(name.release) >= 18) { + /* Only use MAP_JIT if a hardened runtime is used */ + + ptr = mmap(NULL, page_size, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON, -1, 0); + + if (ptr == MAP_FAILED) { + map_jit_flag = MAP_JIT; + } else { + munmap(ptr, page_size); + } + } } return map_jit_flag; @@ -131,12 +148,13 @@ static SLJIT_INLINE int get_map_jit_flag() } #endif /* MAP_JIT */ - +#endif /* MAP_ANON */ #endif /* __APPLE__ */ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) { void *retval; + const int prot = PROT_READ | PROT_WRITE | PROT_EXEC; #ifdef MAP_ANON @@ -146,16 +164,24 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) flags |= get_map_jit_flag(); #endif - retval = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC, flags, -1, 0); + retval = mmap(NULL, size, prot, flags, -1, 0); #else /* !MAP_ANON */ - if (dev_zero < 0) { - if (open_dev_zero()) - return NULL; - } - retval = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE, dev_zero, 0); + if (SLJIT_UNLIKELY((dev_zero < 0) && open_dev_zero())) + return NULL; + + retval = mmap(NULL, size, prot, MAP_PRIVATE, dev_zero, 0); #endif /* MAP_ANON */ - return (retval != MAP_FAILED) ? retval : NULL; + if (retval == MAP_FAILED) + retval = NULL; + else { + if (mprotect(retval, size, prot) < 0) { + munmap(retval, size); + retval = NULL; + } + } + + return retval; } static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) @@ -226,7 +252,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) struct free_block *free_block; sljit_uw chunk_size; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); if (size < (64 - sizeof(struct block_header))) size = (64 - sizeof(struct block_header)); size = ALIGN_SIZE(size); @@ -250,7 +276,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } allocated_size += size; header->size = size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } free_block = free_block->next; @@ -259,7 +285,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) chunk_size = (size + sizeof(struct block_header) + CHUNK_SIZE - 1) & CHUNK_MASK; header = (struct block_header*)alloc_chunk(chunk_size); if (!header) { - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return NULL; } @@ -286,7 +312,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } next_header->size = 1; next_header->prev_size = chunk_size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } @@ -295,7 +321,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) struct block_header *header; struct free_block* free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); header = AS_BLOCK_HEADER(ptr, -(sljit_sw)sizeof(struct block_header)); allocated_size -= header->size; @@ -332,7 +358,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) } } - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) @@ -340,7 +366,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) struct free_block* free_block; struct free_block* next_free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); free_block = free_blocks; while (free_block) { @@ -355,5 +381,5 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) } SLJIT_ASSERT((total_size && free_blocks) || (!total_size && !free_blocks)); - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } diff --git a/dist2/src/sljit/sljitLir.c b/dist2/src/sljit/sljitLir.c index ded9541b..d817c90b 100644 --- a/dist2/src/sljit/sljitLir.c +++ b/dist2/src/sljit/sljitLir.c @@ -28,7 +28,6 @@ #ifdef _WIN32 -/* For SLJIT_CACHE_FLUSH, which can expand to FlushInstructionCache. */ #include <windows.h> #endif /* _WIN32 */ @@ -144,6 +143,7 @@ #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) # define PATCH_MD 0x10 #endif +# define TYPE_SHIFT 13 #endif #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) @@ -222,14 +222,6 @@ # define FCSR_FCC 33 #endif -#if (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) -# define IS_JAL 0x04 -# define IS_COND 0x08 - -# define PATCH_B 0x10 -# define PATCH_J 0x20 -#endif - #if (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) # define IS_MOVABLE 0x04 # define IS_COND 0x08 @@ -273,6 +265,8 @@ #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) #include "sljitProtExecAllocator.c" +#elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR) +#include "sljitWXExecAllocator.c" #else #include "sljitExecAllocator.c" #endif @@ -285,6 +279,10 @@ #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr)) #endif +#ifndef SLJIT_UPDATE_WX_FLAGS +#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) +#endif + /* Argument checking features. */ #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) @@ -365,7 +363,7 @@ static sljit_s32 compiler_initialized = 0; static void init_compiler(void); #endif -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data) +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data) { struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data); if (!compiler) @@ -392,6 +390,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allo compiler->error = SLJIT_SUCCESS; compiler->allocator_data = allocator_data; + compiler->exec_allocator_data = exec_allocator_data; compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data); compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data); @@ -484,22 +483,28 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compi } #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { + SLJIT_UNUSED_ARG(exec_allocator_data); + /* Remove thumb mode flag. */ - SLJIT_FREE_EXEC((void*)((sljit_uw)code & ~0x1)); + SLJIT_FREE_EXEC((void*)((sljit_uw)code & ~0x1), exec_allocator_data); } #elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { + SLJIT_UNUSED_ARG(exec_allocator_data); + /* Resolve indirection. */ code = (void*)(*(sljit_uw*)code); - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, exec_allocator_data); } #else -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { - SLJIT_FREE_EXEC(code); + SLJIT_UNUSED_ARG(exec_allocator_data); + + SLJIT_FREE_EXEC(code, exec_allocator_data); } #endif @@ -521,6 +526,12 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw } } +SLJIT_API_FUNC_ATTRIBUTE void sljit_set_put_label(struct sljit_put_label *put_label, struct sljit_label *label) +{ + if (SLJIT_LIKELY(!!put_label)) + put_label->label = label; +} + SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags) { SLJIT_UNUSED_ARG(compiler); @@ -620,6 +631,33 @@ static SLJIT_INLINE sljit_s32 get_arg_count(sljit_s32 arg_types) return arg_count; } + +/* Only used in RISC architectures where the instruction size is constant */ +#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \ + && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) + +static SLJIT_INLINE sljit_uw compute_next_addr(struct sljit_label *label, struct sljit_jump *jump, + struct sljit_const *const_, struct sljit_put_label *put_label) +{ + sljit_uw result = ~(sljit_uw)0; + + if (label) + result = label->size; + + if (jump && jump->addr < result) + result = jump->addr; + + if (const_ && const_->addr < result) + result = const_->addr; + + if (put_label && put_label->addr < result) + result = put_label->addr; + + return result; +} + +#endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_S390X */ + static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler, sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) @@ -687,6 +725,19 @@ static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_comp compiler->last_const = const_; } +static SLJIT_INLINE void set_put_label(struct sljit_put_label *put_label, struct sljit_compiler *compiler, sljit_uw offset) +{ + put_label->next = NULL; + put_label->label = NULL; + put_label->addr = compiler->size - offset; + put_label->flags = 0; + if (compiler->last_put_label) + compiler->last_put_label->next = put_label; + else + compiler->put_labels = put_label; + compiler->last_put_label = put_label; +} + #define ADDRESSING_DEPENDS_ON(exp, reg) \ (((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg)) @@ -882,7 +933,8 @@ static void sljit_verbose_fparam(struct sljit_compiler *compiler, sljit_s32 p, s static const char* op0_names[] = { (char*)"breakpoint", (char*)"nop", (char*)"lmul.uw", (char*)"lmul.sw", - (char*)"divmod.u", (char*)"divmod.s", (char*)"div.u", (char*)"div.s" + (char*)"divmod.u", (char*)"divmod.s", (char*)"div.u", (char*)"div.s", + (char*)"endbr", (char*)"skip_frames_before_return" }; static const char* op1_names[] = { @@ -899,6 +951,12 @@ static const char* op2_names[] = { (char*)"shl", (char*)"lshr", (char*)"ashr", }; +static const char* op_src_names[] = { + (char*)"fast_return", (char*)"skip_frames_before_fast_return", + (char*)"prefetch_l1", (char*)"prefetch_l2", + (char*)"prefetch_l3", (char*)"prefetch_once", +}; + static const char* fop1_names[] = { (char*)"mov", (char*)"conv", (char*)"conv", (char*)"conv", (char*)"conv", (char*)"conv", (char*)"cmp", (char*)"neg", @@ -1108,37 +1166,21 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fast_enter(struct sljit_c CHECK_RETURN_OK; } -static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ -#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) - FUNCTION_CHECK_SRC(src, srcw); - CHECK_ARGUMENT(src != SLJIT_IMM); - compiler->last_flags = 0; -#endif -#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) - if (SLJIT_UNLIKELY(!!compiler->verbose)) { - fprintf(compiler->verbose, " fast_return "); - sljit_verbose_param(compiler, src, srcw); - fprintf(compiler->verbose, "\n"); - } -#endif - CHECK_RETURN_OK; -} - static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op) { #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) CHECK_ARGUMENT((op >= SLJIT_BREAKPOINT && op <= SLJIT_LMUL_SW) - || ((op & ~SLJIT_I32_OP) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_I32_OP) <= SLJIT_DIV_SW)); - CHECK_ARGUMENT(op < SLJIT_LMUL_UW || compiler->scratches >= 2); - if (op >= SLJIT_LMUL_UW) + || ((op & ~SLJIT_I32_OP) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_I32_OP) <= SLJIT_DIV_SW) + || (op >= SLJIT_ENDBR && op <= SLJIT_SKIP_FRAMES_BEFORE_RETURN)); + CHECK_ARGUMENT(GET_OPCODE(op) < SLJIT_LMUL_UW || GET_OPCODE(op) >= SLJIT_ENDBR || compiler->scratches >= 2); + if ((GET_OPCODE(op) >= SLJIT_LMUL_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) || op == SLJIT_SKIP_FRAMES_BEFORE_RETURN) compiler->last_flags = 0; #endif #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) if (SLJIT_UNLIKELY(!!compiler->verbose)) { fprintf(compiler->verbose, " %s", op0_names[GET_OPCODE(op) - SLJIT_OP0_BASE]); - if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW) { + if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) { fprintf(compiler->verbose, (op & SLJIT_I32_OP) ? "32" : "w"); } fprintf(compiler->verbose, "\n"); @@ -1180,7 +1222,7 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op1(struct sljit_compiler break; } - FUNCTION_CHECK_DST(dst, dstw, 1); + FUNCTION_CHECK_DST(dst, dstw, HAS_FLAGS(op)); FUNCTION_CHECK_SRC(src, srcw); if (GET_OPCODE(op) >= SLJIT_NOT) { @@ -1260,7 +1302,7 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler break; } - FUNCTION_CHECK_DST(dst, dstw, 1); + FUNCTION_CHECK_DST(dst, dstw, HAS_FLAGS(op)); FUNCTION_CHECK_SRC(src1, src1w); FUNCTION_CHECK_SRC(src2, src2w); compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_I32_OP | SLJIT_SET_Z)); @@ -1281,6 +1323,33 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler CHECK_RETURN_OK; } +static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ +#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) + CHECK_ARGUMENT(op >= SLJIT_FAST_RETURN && op <= SLJIT_PREFETCH_ONCE); + FUNCTION_CHECK_SRC(src, srcw); + + if (op == SLJIT_FAST_RETURN || op == SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN) + { + CHECK_ARGUMENT(src != SLJIT_IMM); + compiler->last_flags = 0; + } + else if (op >= SLJIT_PREFETCH_L1 && op <= SLJIT_PREFETCH_ONCE) + { + CHECK_ARGUMENT(src & SLJIT_MEM); + } +#endif +#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) + if (SLJIT_UNLIKELY(!!compiler->verbose)) { + fprintf(compiler->verbose, " %s ", op_src_names[op - SLJIT_OP_SRC_BASE]); + sljit_verbose_param(compiler, src, srcw); + fprintf(compiler->verbose, "\n"); + } +#endif + CHECK_RETURN_OK; +} + static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_register_index(sljit_s32 reg) { SLJIT_UNUSED_ARG(reg); @@ -1316,6 +1385,8 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_custom(struct sljit_co #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) CHECK_ARGUMENT((size == 2 && (((sljit_sw)instruction) & 0x1) == 0) || (size == 4 && (((sljit_sw)instruction) & 0x3) == 0)); +#elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) + CHECK_ARGUMENT(size == 2 || size == 4 || size == 6); #else CHECK_ARGUMENT(size == 4 && (((sljit_sw)instruction) & 0x3) == 0); #endif @@ -1905,6 +1976,21 @@ static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_const(struct sljit_compil CHECK_RETURN_OK; } +static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ +#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) + FUNCTION_CHECK_DST(dst, dstw, 0); +#endif +#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) + if (SLJIT_UNLIKELY(!!compiler->verbose)) { + fprintf(compiler->verbose, " put_label "); + sljit_verbose_param(compiler, dst, dstw); + fprintf(compiler->verbose, "\n"); + } +#endif + CHECK_RETURN_OK; +} + #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */ #define SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw) \ @@ -1957,7 +2043,7 @@ static SLJIT_INLINE sljit_s32 emit_mov_before_return(struct sljit_compiler *comp #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \ || (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC) \ || (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) \ - || ((defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) && !(defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1)) + || ((defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) && !(defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1 && SLJIT_MIPS_REV < 6)) static SLJIT_INLINE sljit_s32 sljit_emit_cmov_generic(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 dst_reg, @@ -2034,8 +2120,8 @@ static SLJIT_INLINE sljit_s32 sljit_emit_cmov_generic(struct sljit_compiler *com # include "sljitNativeMIPS_common.c" #elif (defined SLJIT_CONFIG_SPARC && SLJIT_CONFIG_SPARC) # include "sljitNativeSPARC_common.c" -#elif (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) -# include "sljitNativeTILEGX_64.c" +#elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) +# include "sljitNativeS390X.c" #endif #if !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) @@ -2066,7 +2152,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler #endif if (SLJIT_UNLIKELY((src1 & SLJIT_IMM) && !(src2 & SLJIT_IMM))) { - /* Immediate is prefered as second argument by most architectures. */ + /* Immediate is preferred as second argument by most architectures. */ switch (condition) { case SLJIT_LESS: condition = SLJIT_GREATER; @@ -2215,9 +2301,10 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) return "unsupported"; } -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data) +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data) { SLJIT_UNUSED_ARG(allocator_data); + SLJIT_UNUSED_ARG(exec_allocator_data); SLJIT_UNREACHABLE(); return NULL; } @@ -2265,9 +2352,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) return 0; } -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { SLJIT_UNUSED_ARG(code); + SLJIT_UNUSED_ARG(exec_allocator_data); SLJIT_UNREACHABLE(); } @@ -2322,15 +2410,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return SLJIT_ERR_UNSUPPORTED; } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - SLJIT_UNUSED_ARG(compiler); - SLJIT_UNUSED_ARG(src); - SLJIT_UNUSED_ARG(srcw); - SLJIT_UNREACHABLE(); - return SLJIT_ERR_UNSUPPORTED; -} - SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op) { SLJIT_UNUSED_ARG(compiler); @@ -2370,6 +2449,17 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_ERR_UNSUPPORTED; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + SLJIT_UNUSED_ARG(compiler); + SLJIT_UNUSED_ARG(op); + SLJIT_UNUSED_ARG(src); + SLJIT_UNUSED_ARG(srcw); + SLJIT_UNREACHABLE(); + return SLJIT_ERR_UNSUPPORTED; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { SLJIT_UNREACHABLE(); @@ -2490,6 +2580,13 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw SLJIT_UNREACHABLE(); } +SLJIT_API_FUNC_ATTRIBUTE void sljit_set_put_label(struct sljit_put_label *put_label, struct sljit_label *label) +{ + SLJIT_UNUSED_ARG(put_label); + SLJIT_UNUSED_ARG(label); + SLJIT_UNREACHABLE(); +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw) { SLJIT_UNUSED_ARG(compiler); @@ -2581,6 +2678,14 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi return NULL; } +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + SLJIT_UNUSED_ARG(compiler); + SLJIT_UNUSED_ARG(dst); + SLJIT_UNUSED_ARG(dstw); + return NULL; +} + SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { SLJIT_UNUSED_ARG(addr); @@ -2597,4 +2702,4 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_consta SLJIT_UNREACHABLE(); } -#endif +#endif /* !SLJIT_CONFIG_UNSUPPORTED */ diff --git a/dist2/src/sljit/sljitLir.h b/dist2/src/sljit/sljitLir.h index e71890cf..93d28046 100644 --- a/dist2/src/sljit/sljitLir.h +++ b/dist2/src/sljit/sljitLir.h @@ -24,8 +24,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_LIR_H_ -#define _SLJIT_LIR_H_ +#ifndef SLJIT_LIR_H_ +#define SLJIT_LIR_H_ /* ------------------------------------------------------------------------ @@ -70,9 +70,11 @@ - pass --smc-check=all argument to valgrind, since JIT is a "self-modifying code" */ -#if !(defined SLJIT_NO_DEFAULT_CONFIG && SLJIT_NO_DEFAULT_CONFIG) +#if (defined SLJIT_HAVE_CONFIG_PRE && SLJIT_HAVE_CONFIG_PRE) +#include "sljitConfigPre.h" +#endif /* SLJIT_HAVE_CONFIG_PRE */ + #include "sljitConfig.h" -#endif /* The following header file defines useful macros for fine tuning sljit based code generators. They are listed in the beginning @@ -80,6 +82,14 @@ of sljitConfigInternal.h */ #include "sljitConfigInternal.h" +#if (defined SLJIT_HAVE_CONFIG_POST && SLJIT_HAVE_CONFIG_POST) +#include "sljitConfigPost.h" +#endif /* SLJIT_HAVE_CONFIG_POST */ + +#ifdef __cplusplus +extern "C" { +#endif + /* --------------------------------------------------------------------- */ /* Error codes */ /* --------------------------------------------------------------------- */ @@ -154,10 +164,10 @@ of sljitConfigInternal.h */ */ /* When SLJIT_UNUSED is specified as the destination of sljit_emit_op1 - or sljit_emit_op2 operations the result is discarded. If no status - flags are set, no instructions are emitted for these operations. Data - prefetch is a special exception, see SLJIT_MOV operation. Other SLJIT - operations do not support SLJIT_UNUSED as a destination operand. */ + or sljit_emit_op2 operations the result is discarded. Some status + flags must be set when the destination is SLJIT_UNUSED, because the + operation would have no effect otherwise. Other SLJIT operations do + not support SLJIT_UNUSED as a destination operand. */ #define SLJIT_UNUSED 0 /* Scratch registers. */ @@ -348,13 +358,20 @@ struct sljit_label { struct sljit_jump { struct sljit_jump *next; sljit_uw addr; - sljit_sw flags; + sljit_uw flags; union { sljit_uw target; - struct sljit_label* label; + struct sljit_label *label; } u; }; +struct sljit_put_label { + struct sljit_put_label *next; + struct sljit_label *label; + sljit_uw addr; + sljit_uw flags; +}; + struct sljit_const { struct sljit_const *next; sljit_uw addr; @@ -366,12 +383,15 @@ struct sljit_compiler { struct sljit_label *labels; struct sljit_jump *jumps; + struct sljit_put_label *put_labels; struct sljit_const *consts; struct sljit_label *last_label; struct sljit_jump *last_jump; struct sljit_const *last_const; + struct sljit_put_label *last_put_label; void *allocator_data; + void *exec_allocator_data; struct sljit_memory_fragment *buf; struct sljit_memory_fragment *abuf; @@ -438,9 +458,9 @@ struct sljit_compiler { sljit_sw cache_argw; #endif -#if (defined SLJIT_CONFIG_TILEGX && SLJIT_CONFIG_TILEGX) - sljit_s32 cache_arg; - sljit_sw cache_argw; +#if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) + /* Need to allocate register save area to make calls. */ + sljit_s32 have_save_area; #endif #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) @@ -472,10 +492,12 @@ struct sljit_compiler { custom memory managers. This pointer is passed to SLJIT_MALLOC and SLJIT_FREE macros. Most allocators (including the default one) ignores this value, and it is recommended to pass NULL - as a dummy value for allocator_data. + as a dummy value for allocator_data. The exec_allocator_data + has the same purpose but this one is passed to SLJIT_MALLOC_EXEC / + SLJIT_MALLOC_FREE functions. Returns NULL if failed. */ -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data); +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data); /* Frees everything except the compiled machine code. */ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler); @@ -522,7 +544,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil /* Free executable code. */ -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code); +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data); /* When the protected executable allocator is used the JIT code is mapped @@ -558,10 +580,14 @@ static SLJIT_INLINE sljit_uw sljit_get_generated_code_size(struct sljit_compiler #define SLJIT_HAS_FPU 0 /* [Limitation] Some registers are virtual registers. */ #define SLJIT_HAS_VIRTUAL_REGISTERS 1 +/* [Emulated] Has zero register (setting a memory location to zero is efficient). */ +#define SLJIT_HAS_ZERO_REGISTER 2 /* [Emulated] Count leading zero is supported. */ -#define SLJIT_HAS_CLZ 2 +#define SLJIT_HAS_CLZ 3 +/* [Emulated] Conditional move is supported. */ +#define SLJIT_HAS_CMOV 4 /* [Emulated] Conditional move is supported. */ -#define SLJIT_HAS_CMOV 3 +#define SLJIT_HAS_PREFETCH 5 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) /* [Not emulated] SSE2 support is available on x86. */ @@ -649,10 +675,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *comp sljit_s32 src, sljit_sw srcw); /* Generating entry and exit points for fast call functions (see SLJIT_FAST_CALL). - Both sljit_emit_fast_enter and sljit_emit_fast_return functions preserve the + Both sljit_emit_fast_enter and SLJIT_FAST_RETURN operations preserve the values of all registers and stack frame. The return address is stored in the dst argument of sljit_emit_fast_enter, and this return address can be passed - to sljit_emit_fast_return to continue the execution after the fast call. + to SLJIT_FAST_RETURN to continue the execution after the fast call. Fast calls are cheap operations (usually only a single call instruction is emitted) but they do not preserve any registers. However the callee function @@ -660,16 +686,15 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *comp efficiently exploited by various optimizations. Registers can be saved manually by the callee function if needed. - Although returning to different address by sljit_emit_fast_return is possible, + Although returning to different address by SLJIT_FAST_RETURN is possible, this address usually cannot be predicted by the return address predictor of - modern CPUs which may reduce performance. Furthermore using sljit_emit_ijump - to return is also inefficient since return address prediction is usually - triggered by a specific form of ijump. + modern CPUs which may reduce performance. Furthermore certain security + enhancement technologies such as Intel Control-flow Enforcement Technology + (CET) may disallow returning to a different address. Flags: - (does not modify flags). */ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw); -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw); /* Source and destination operands for arithmetical instructions @@ -683,7 +708,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler */ /* - IMPORATNT NOTE: memory access MUST be naturally aligned except + IMPORTANT NOTE: memory access MUST be naturally aligned unless SLJIT_UNALIGNED macro is defined and its value is 1. length | alignment @@ -725,6 +750,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler mips: [reg+imm], -65536 <= imm <= 65535 sparc: [reg+imm], -4096 <= imm <= 4095 [reg+reg] is supported + s390x: [reg+imm], -2^19 <= imm < 2^19 + [reg+reg] is supported + Write-back is not supported */ /* Macros for specifying operand types. */ @@ -878,6 +906,14 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler the behaviour is undefined. */ #define SLJIT_DIV_SW (SLJIT_OP0_BASE + 7) #define SLJIT_DIV_S32 (SLJIT_DIV_SW | SLJIT_I32_OP) +/* Flags: - (does not modify flags) + ENDBR32 instruction for x86-32 and ENDBR64 instruction for x86-64 + when Intel Control-flow Enforcement Technology (CET) is enabled. + No instruction for other architectures. */ +#define SLJIT_ENDBR (SLJIT_OP0_BASE + 8) +/* Flags: - (may destroy flags) + Skip stack frames before return. */ +#define SLJIT_SKIP_FRAMES_BEFORE_RETURN (SLJIT_OP0_BASE + 9) SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op); @@ -895,15 +931,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile U32 - unsigned int (32 bit) data transfer S32 - signed int (32 bit) data transfer P - pointer (sljit_p) data transfer - - If the destination of a MOV instruction is SLJIT_UNUSED and the source - operand is a memory address the compiler emits a prefetch instruction - if this instruction is supported by the current CPU. Higher data sizes - bring the data closer to the core: a MOV with word size loads the data - into a higher level cache than a byte size. Otherwise the type does not - affect the prefetch instruction. Furthermore a prefetch instruction - never fails, so it can be used to prefetch a data from an address and - check whether that address is NULL afterwards. */ /* Flags: - (does not modify flags) */ @@ -1008,8 +1035,46 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile sljit_s32 src1, sljit_sw src1w, sljit_s32 src2, sljit_sw src2w); +/* Starting index of opcodes for sljit_emit_op2. */ +#define SLJIT_OP_SRC_BASE 128 + +/* Note: src cannot be an immedate value + Flags: - (does not modify flags) */ +#define SLJIT_FAST_RETURN (SLJIT_OP_SRC_BASE + 0) +/* Skip stack frames before fast return. + Note: src cannot be an immedate value + Flags: may destroy flags. */ +#define SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN (SLJIT_OP_SRC_BASE + 1) +/* Prefetch value into the level 1 data cache + Note: if the target CPU does not support data prefetch, + no instructions are emitted. + Note: this instruction never fails, even if the memory address is invalid. + Flags: - (does not modify flags) */ +#define SLJIT_PREFETCH_L1 (SLJIT_OP_SRC_BASE + 2) +/* Prefetch value into the level 2 data cache + Note: same as SLJIT_PREFETCH_L1 if the target CPU + does not support this instruction form. + Note: this instruction never fails, even if the memory address is invalid. + Flags: - (does not modify flags) */ +#define SLJIT_PREFETCH_L2 (SLJIT_OP_SRC_BASE + 3) +/* Prefetch value into the level 3 data cache + Note: same as SLJIT_PREFETCH_L2 if the target CPU + does not support this instruction form. + Note: this instruction never fails, even if the memory address is invalid. + Flags: - (does not modify flags) */ +#define SLJIT_PREFETCH_L3 (SLJIT_OP_SRC_BASE + 4) +/* Prefetch a value which is only used once (and can be discarded afterwards) + Note: same as SLJIT_PREFETCH_L1 if the target CPU + does not support this instruction form. + Note: this instruction never fails, even if the memory address is invalid. + Flags: - (does not modify flags) */ +#define SLJIT_PREFETCH_ONCE (SLJIT_OP_SRC_BASE + 5) + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw); + /* Starting index of opcodes for sljit_emit_fop1. */ -#define SLJIT_FOP1_BASE 128 +#define SLJIT_FOP1_BASE 160 /* Flags: - (does not modify flags) */ #define SLJIT_MOV_F64 (SLJIT_FOP1_BASE + 0) @@ -1048,7 +1113,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compil sljit_s32 src, sljit_sw srcw); /* Starting index of opcodes for sljit_emit_fop2. */ -#define SLJIT_FOP2_BASE 160 +#define SLJIT_FOP2_BASE 192 /* Flags: - (does not modify flags) */ #define SLJIT_ADD_F64 (SLJIT_FOP2_BASE + 0) @@ -1152,7 +1217,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi /* Unconditional jump types. */ #define SLJIT_JUMP 24 - /* Fast calling method. See sljit_emit_fast_enter / sljit_emit_fast_return. */ + /* Fast calling method. See sljit_emit_fast_enter / SLJIT_FAST_RETURN. */ #define SLJIT_FAST_CALL 25 /* Called function must be declared with the SLJIT_FUNC attribute. */ #define SLJIT_CALL 26 @@ -1314,10 +1379,17 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compil Flags: - (may destroy flags) */ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset); -/* The constant can be changed runtime (see: sljit_set_const) +/* Store a value that can be changed runtime (see: sljit_get_const_addr / sljit_set_const) Flags: - (does not modify flags) */ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value); +/* Store the value of a label (see: sljit_set_put_label) + Flags: - (does not modify flags) */ +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw); + +/* Set the value stored by put_label to this label. */ +SLJIT_API_FUNC_ATTRIBUTE void sljit_set_put_label(struct sljit_put_label *put_label, struct sljit_label *label); + /* After the code generation the address for label, jump and const instructions are computed. Since these structures are freed by sljit_free_compiler, the addresses must be preserved by the user program elsewere. */ @@ -1345,12 +1417,6 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void); /* Portable helper function to get an offset of a member. */ #define SLJIT_OFFSETOF(base, member) ((sljit_sw)(&((base*)0x10)->member) - 0x10) -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) -/* This global lock is useful to compile common functions. */ -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void); -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void); -#endif - #if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) /* The sljit_stack structure and its manipulation functions provides @@ -1474,4 +1540,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *c SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags); -#endif /* _SLJIT_LIR_H_ */ +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* SLJIT_LIR_H_ */ diff --git a/dist2/src/sljit/sljitNativeARM_32.c b/dist2/src/sljit/sljitNativeARM_32.c index 6d61eed9..ae8479f0 100644 --- a/dist2/src/sljit/sljitNativeARM_32.c +++ b/dist2/src/sljit/sljitNativeARM_32.c @@ -467,18 +467,28 @@ static SLJIT_INLINE void inline_set_jump_addr(sljit_uw jump_ptr, sljit_sw execut sljit_s32 bl = (mov_pc & 0x0000f000) != RD(TMP_PC); sljit_sw diff = (sljit_sw)(((sljit_sw)new_addr - (sljit_sw)(inst + 2) - executable_offset) >> 2); + SLJIT_UNUSED_ARG(executable_offset); + if (diff <= 0x7fffff && diff >= -0x800000) { /* Turn to branch. */ if (!bl) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } inst[0] = (mov_pc & COND_MASK) | (B - CONDITIONAL) | (diff & 0xffffff); if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } else { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } inst[0] = (mov_pc & COND_MASK) | (BL - CONDITIONAL) | (diff & 0xffffff); inst[1] = NOP; if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -491,28 +501,52 @@ static SLJIT_INLINE void inline_set_jump_addr(sljit_uw jump_ptr, sljit_sw execut ptr = inst + 1; if (*inst != mov_pc) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + (!bl ? 1 : 2), 0); + } inst[0] = mov_pc; if (!bl) { if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } else { inst[1] = BLX | RM(TMP_REG1); if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } } } + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0); + } + *ptr = new_addr; + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1); + } } #else sljit_uw *inst = (sljit_uw*)jump_ptr; + + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT); + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } + inst[0] = MOVW | (inst[0] & 0xf000) | ((new_addr << 4) & 0xf0000) | (new_addr & 0xfff); inst[1] = MOVT | (inst[1] & 0xf000) | ((new_addr >> 12) & 0xf0000) | ((new_addr >> 16) & 0xfff); + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -529,10 +563,18 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off sljit_uw ldr_literal = ptr[1]; sljit_uw src2; + SLJIT_UNUSED_ARG(executable_offset); + src2 = get_imm(new_constant); if (src2) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = 0xe3a00000 | (ldr_literal & 0xf000) | src2; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } @@ -541,8 +583,14 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off src2 = get_imm(~new_constant); if (src2) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = 0xe3e00000 | (ldr_literal & 0xf000) | src2; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } @@ -555,19 +603,44 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off ptr = inst + 1; if (*inst != ldr_literal) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = ldr_literal; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0); + } + *ptr = new_constant; + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1); + } #else sljit_uw *inst = (sljit_uw*)addr; + + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT); + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } + inst[0] = MOVW | (inst[0] & 0xf000) | ((new_constant << 4) & 0xf0000) | (new_constant & 0xfff); inst[1] = MOVT | (inst[1] & 0xf000) | ((new_constant >> 12) & 0xf0000) | ((new_constant >> 16) & 0xfff); + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -583,8 +656,9 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_uw *buf_end; sljit_uw size; sljit_uw word_count; + sljit_uw next_addr; sljit_sw executable_offset; - sljit_sw jump_addr; + sljit_sw addr; #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) sljit_uw cpool_size; sljit_uw cpool_skip_alignment; @@ -597,6 +671,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); @@ -610,7 +685,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil #else size = compiler->size; #endif - code = (sljit_uw*)SLJIT_MALLOC_EXEC(size * sizeof(sljit_uw)); + code = (sljit_uw*)SLJIT_MALLOC_EXEC(size * sizeof(sljit_uw), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -625,11 +700,13 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = code; word_count = 0; + next_addr = 1; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; if (label && label->size == 0) { label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset); @@ -649,7 +726,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil } else { if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) { - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, compiler->exec_allocator_data); compiler->error = SLJIT_ERR_ALLOC_FAILED; return NULL; } @@ -662,6 +739,8 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); label->size = code_ptr - code; label = label->next; + + next_addr = compute_next_addr(label, jump, const_, put_label); } } } @@ -669,35 +748,45 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil else if ((*buf_ptr & 0xff000000) != PUSH_POOL) { #endif *code_ptr = *buf_ptr++; + if (next_addr == word_count) { + SLJIT_ASSERT(!label || label->size >= word_count); + SLJIT_ASSERT(!jump || jump->addr >= word_count); + SLJIT_ASSERT(!const_ || const_->addr >= word_count); + SLJIT_ASSERT(!put_label || put_label->addr >= word_count); + /* These structures are ordered by their address. */ - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - if (jump && jump->addr == word_count) { + if (jump && jump->addr == word_count) { #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) - if (detect_jump_type(jump, code_ptr, code, executable_offset)) - code_ptr--; - jump->addr = (sljit_uw)code_ptr; + if (detect_jump_type(jump, code_ptr, code, executable_offset)) + code_ptr--; + jump->addr = (sljit_uw)code_ptr; #else - jump->addr = (sljit_uw)(code_ptr - 2); - if (detect_jump_type(jump, code_ptr, code, executable_offset)) - code_ptr -= 2; + jump->addr = (sljit_uw)(code_ptr - 2); + if (detect_jump_type(jump, code_ptr, code, executable_offset)) + code_ptr -= 2; #endif - jump = jump->next; - } - if (label && label->size == word_count) { - /* code_ptr can be affected above. */ - label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr + 1, executable_offset); - label->size = (code_ptr + 1) - code; - label = label->next; - } - if (const_ && const_->addr == word_count) { + jump = jump->next; + } + if (label && label->size == word_count) { + /* code_ptr can be affected above. */ + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr + 1, executable_offset); + label->size = (code_ptr + 1) - code; + label = label->next; + } + if (const_ && const_->addr == word_count) { #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) - const_->addr = (sljit_uw)code_ptr; + const_->addr = (sljit_uw)code_ptr; #else - const_->addr = (sljit_uw)(code_ptr - 1); + const_->addr = (sljit_uw)(code_ptr - 1); #endif - const_ = const_->next; + const_ = const_->next; + } + if (put_label && put_label->addr == word_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr++; #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) @@ -725,6 +814,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) SLJIT_ASSERT(cpool_size == 0); @@ -739,7 +829,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil cpool_current_index = 0; while (buf_ptr < buf_end) { if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) { - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, compiler->exec_allocator_data); compiler->error = SLJIT_ERR_ALLOC_FAILED; return NULL; } @@ -755,15 +845,15 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil buf_ptr = (sljit_uw *)jump->addr; if (jump->flags & PATCH_B) { - jump_addr = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr + 2, executable_offset); + addr = (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr + 2, executable_offset); if (!(jump->flags & JUMP_ADDR)) { SLJIT_ASSERT(jump->flags & JUMP_LABEL); - SLJIT_ASSERT(((sljit_sw)jump->u.label->addr - jump_addr) <= 0x01ffffff && ((sljit_sw)jump->u.label->addr - jump_addr) >= -0x02000000); - *buf_ptr |= (((sljit_sw)jump->u.label->addr - jump_addr) >> 2) & 0x00ffffff; + SLJIT_ASSERT(((sljit_sw)jump->u.label->addr - addr) <= 0x01ffffff && ((sljit_sw)jump->u.label->addr - addr) >= -0x02000000); + *buf_ptr |= (((sljit_sw)jump->u.label->addr - addr) >> 2) & 0x00ffffff; } else { - SLJIT_ASSERT(((sljit_sw)jump->u.target - jump_addr) <= 0x01ffffff && ((sljit_sw)jump->u.target - jump_addr) >= -0x02000000); - *buf_ptr |= (((sljit_sw)jump->u.target - jump_addr) >> 2) & 0x00ffffff; + SLJIT_ASSERT(((sljit_sw)jump->u.target - addr) <= 0x01ffffff && ((sljit_sw)jump->u.target - addr) >= -0x02000000); + *buf_ptr |= (((sljit_sw)jump->u.target - addr) >> 2) & 0x00ffffff; } } else if (jump->flags & SLJIT_REWRITABLE_JUMP) { @@ -813,6 +903,22 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil } #endif + put_label = compiler->put_labels; + while (put_label) { + addr = put_label->label->addr; + buf_ptr = (sljit_uw*)put_label->addr; + +#if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) + SLJIT_ASSERT((buf_ptr[0] & 0xffff0000) == 0xe59f0000); + buf_ptr[((buf_ptr[0] & 0xfff) >> 2) + 2] = addr; +#else + SLJIT_ASSERT((buf_ptr[-1] & 0xfff00000) == MOVW && (buf_ptr[0] & 0xfff00000) == MOVT); + buf_ptr[-1] |= ((addr << 4) & 0xf0000) | (addr & 0xfff); + buf_ptr[0] |= ((addr >> 12) & 0xf0000) | ((addr >> 16) & 0xfff); +#endif + put_label = put_label->next; + } + SLJIT_ASSERT(code_ptr - code <= (sljit_s32)size); compiler->error = SLJIT_ERR_COMPILED; @@ -823,6 +929,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -839,6 +946,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) case SLJIT_HAS_CLZ: case SLJIT_HAS_CMOV: +#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) + case SLJIT_HAS_PREFETCH: +#endif return 1; default: @@ -1645,6 +1755,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile | (saved_reg_list[0] << 12) /* ldr rX, [sp], #8/16 */); } return SLJIT_SUCCESS; + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; @@ -1659,14 +1772,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { -#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) - return emit_op_mem(compiler, PRELOAD | LOAD_DATA, TMP_PC, src, srcw, TMP_REG1); -#endif - return SLJIT_SUCCESS; - } - switch (GET_OPCODE(op)) { case SLJIT_MOV: case SLJIT_MOV_U32: @@ -1748,6 +1853,40 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_SUCCESS; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + SLJIT_ASSERT(reg_map[TMP_REG2] == 14); + + if (FAST_IS_REG(src)) + FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG2) | RM(src))); + else + FAIL_IF(emit_op_mem(compiler, WORD_SIZE | LOAD_DATA, TMP_REG2, src, srcw, TMP_REG1)); + + return push_inst(compiler, BX | RM(TMP_REG2)); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: +#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7) + SLJIT_ASSERT(src & SLJIT_MEM); + return emit_op_mem(compiler, PRELOAD | LOAD_DATA, TMP_PC, src, srcw, TMP_REG1); +#else /* !SLJIT_CONFIG_ARM_V7 */ + return SLJIT_SUCCESS; +#endif /* SLJIT_CONFIG_ARM_V7 */ + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -2010,22 +2149,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return emit_op_mem(compiler, WORD_SIZE, TMP_REG2, dst, dstw, TMP_REG1); } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - SLJIT_ASSERT(reg_map[TMP_REG2] == 14); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG2) | RM(src))); - else - FAIL_IF(emit_op_mem(compiler, WORD_SIZE | LOAD_DATA, TMP_REG2, src, srcw, TMP_REG1)); - - return push_inst(compiler, BX | RM(TMP_REG2)); -} - /* --------------------------------------------------------------------- */ /* Conditional instructions */ /* --------------------------------------------------------------------- */ @@ -2584,11 +2707,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compile } else { if (is_type1_transfer) { - if (memw > 4095 && memw < -4095) + if (memw > 4095 || memw < -4095) return SLJIT_ERR_UNSUPPORTED; } else { - if (memw > 255 && memw < -255) + if (memw > 255 || memw < -255) return SLJIT_ERR_UNSUPPORTED; } } @@ -2639,28 +2762,55 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compile SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) { struct sljit_const *const_; - sljit_s32 reg; + sljit_s32 dst_r; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); ADJUST_LOCAL_OFFSET(dst, dstw); + dst_r = SLOW_IS_REG(dst) ? dst : TMP_REG2; + +#if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) + PTR_FAIL_IF(push_inst_with_unique_literal(compiler, EMIT_DATA_TRANSFER(WORD_SIZE | LOAD_DATA, 1, dst_r, TMP_PC, 0), init_value)); + compiler->patches++; +#else + PTR_FAIL_IF(emit_imm(compiler, dst_r, init_value)); +#endif + const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const)); PTR_FAIL_IF(!const_); + set_const(const_, compiler); + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, dst, dstw, TMP_REG1)); + return const_; +} - reg = SLOW_IS_REG(dst) ? dst : TMP_REG2; +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + dst_r = SLOW_IS_REG(dst) ? dst : TMP_REG2; #if (defined SLJIT_CONFIG_ARM_V5 && SLJIT_CONFIG_ARM_V5) - PTR_FAIL_IF(push_inst_with_unique_literal(compiler, EMIT_DATA_TRANSFER(WORD_SIZE | LOAD_DATA, 1, reg, TMP_PC, 0), init_value)); + PTR_FAIL_IF(push_inst_with_unique_literal(compiler, EMIT_DATA_TRANSFER(WORD_SIZE | LOAD_DATA, 1, dst_r, TMP_PC, 0), 0)); compiler->patches++; #else - PTR_FAIL_IF(emit_imm(compiler, reg, init_value)); + PTR_FAIL_IF(emit_imm(compiler, dst_r, 0)); #endif - set_const(const_, compiler); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); if (dst & SLJIT_MEM) PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, dst, dstw, TMP_REG1)); - return const_; + return put_label; } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) diff --git a/dist2/src/sljit/sljitNativeARM_64.c b/dist2/src/sljit/sljitNativeARM_64.c index b015695c..52267e7d 100644 --- a/dist2/src/sljit/sljitNativeARM_64.c +++ b/dist2/src/sljit/sljitNativeARM_64.c @@ -151,17 +151,7 @@ static SLJIT_INLINE sljit_s32 emit_imm64_const(struct sljit_compiler *compiler, return push_inst(compiler, MOVK | RD(dst) | ((imm >> 48) << 5) | (3 << 21)); } -static SLJIT_INLINE void modify_imm64_const(sljit_ins* inst, sljit_uw new_imm) -{ - sljit_s32 dst = inst[0] & 0x1f; - SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21))); - inst[0] = MOVZ | dst | ((new_imm & 0xffff) << 5); - inst[1] = MOVK | dst | (((new_imm >> 16) & 0xffff) << 5) | (1 << 21); - inst[2] = MOVK | dst | (((new_imm >> 32) & 0xffff) << 5) | (2 << 21); - inst[3] = MOVK | dst | ((new_imm >> 48) << 5) | (3 << 21); -} - -static SLJIT_INLINE sljit_s32 detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset) +static SLJIT_INLINE sljit_sw detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset) { sljit_sw diff; sljit_uw target_addr; @@ -196,14 +186,14 @@ static SLJIT_INLINE sljit_s32 detect_jump_type(struct sljit_jump *jump, sljit_in return 4; } - if (target_addr <= 0xffffffffl) { + if (target_addr < 0x100000000l) { if (jump->flags & IS_COND) code_ptr[-5] -= (2 << 5); code_ptr[-2] = code_ptr[0]; return 2; } - if (target_addr <= 0xffffffffffffl) { + if (target_addr < 0x1000000000000l) { if (jump->flags & IS_COND) code_ptr[-5] -= (1 << 5); jump->flags |= PATCH_ABS48; @@ -215,6 +205,22 @@ static SLJIT_INLINE sljit_s32 detect_jump_type(struct sljit_jump *jump, sljit_in return 0; } +static SLJIT_INLINE sljit_sw put_label_get_length(struct sljit_put_label *put_label, sljit_uw max_label) +{ + if (max_label < 0x100000000l) { + put_label->flags = 0; + return 2; + } + + if (max_label < 0x1000000000000l) { + put_label->flags = 1; + return 1; + } + + put_label->flags = 2; + return 0; +} + SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler) { struct sljit_memory_fragment *buf; @@ -223,6 +229,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_ins *buf_ptr; sljit_ins *buf_end; sljit_uw word_count; + sljit_uw next_addr; sljit_sw executable_offset; sljit_uw addr; sljit_s32 dst; @@ -230,45 +237,59 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; code_ptr = code; word_count = 0; + next_addr = 0; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; do { buf_ptr = (sljit_ins*)buf->memory; buf_end = buf_ptr + (buf->used_size >> 2); do { *code_ptr = *buf_ptr++; - /* These structures are ordered by their address. */ - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - if (label && label->size == word_count) { - label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); - label->size = code_ptr - code; - label = label->next; - } - if (jump && jump->addr == word_count) { - jump->addr = (sljit_uw)(code_ptr - 4); - code_ptr -= detect_jump_type(jump, code_ptr, code, executable_offset); - jump = jump->next; - } - if (const_ && const_->addr == word_count) { - const_->addr = (sljit_uw)code_ptr; - const_ = const_->next; + if (next_addr == word_count) { + SLJIT_ASSERT(!label || label->size >= word_count); + SLJIT_ASSERT(!jump || jump->addr >= word_count); + SLJIT_ASSERT(!const_ || const_->addr >= word_count); + SLJIT_ASSERT(!put_label || put_label->addr >= word_count); + + /* These structures are ordered by their address. */ + if (label && label->size == word_count) { + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + label->size = code_ptr - code; + label = label->next; + } + if (jump && jump->addr == word_count) { + jump->addr = (sljit_uw)(code_ptr - 4); + code_ptr -= detect_jump_type(jump, code_ptr, code, executable_offset); + jump = jump->next; + } + if (const_ && const_->addr == word_count) { + const_->addr = (sljit_uw)code_ptr; + const_ = const_->next; + } + if (put_label && put_label->addr == word_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)(code_ptr - 3); + code_ptr -= put_label_get_length(put_label, (sljit_uw)(SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size)); + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr ++; word_count ++; @@ -286,6 +307,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size); jump = compiler->jumps; @@ -323,6 +345,23 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } + put_label = compiler->put_labels; + while (put_label) { + addr = put_label->label->addr; + buf_ptr = (sljit_ins *)put_label->addr; + + buf_ptr[0] |= (addr & 0xffff) << 5; + buf_ptr[1] |= ((addr >> 16) & 0xffff) << 5; + + if (put_label->flags >= 1) + buf_ptr[2] |= ((addr >> 32) & 0xffff) << 5; + + if (put_label->flags >= 2) + buf_ptr[3] |= ((addr >> 48) & 0xffff) << 5; + + put_label = put_label->next; + } + compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = (code_ptr - code) * sizeof(sljit_ins); @@ -331,6 +370,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -347,6 +387,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) case SLJIT_HAS_CLZ: case SLJIT_HAS_CMOV: + case SLJIT_HAS_PREFETCH: return 1; default: @@ -1105,6 +1146,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile case SLJIT_DIV_UW: case SLJIT_DIV_SW: return push_inst(compiler, ((op == SLJIT_DIV_UW ? UDIV : SDIV) ^ inv_bits) | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1)); + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; @@ -1122,23 +1166,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) { - SLJIT_ASSERT(reg_map[1] == 0 && reg_map[3] == 2 && reg_map[5] == 4); - - if (op >= SLJIT_MOV_U8 && op <= SLJIT_MOV_S8) - dst = 5; - else if (op >= SLJIT_MOV_U16 && op <= SLJIT_MOV_S16) - dst = 3; - else - dst = 1; - - /* Signed word sized load is the prefetch instruction. */ - return emit_op_mem(compiler, WORD_SIZE | SIGNED, dst, src, srcw, TMP_REG1); - } - return SLJIT_SUCCESS; - } - dst_r = SLOW_IS_REG(dst) ? dst : TMP_REG1; op = GET_OPCODE(op); @@ -1278,6 +1305,46 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_SUCCESS; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + if (FAST_IS_REG(src)) + FAIL_IF(push_inst(compiler, ORR | RD(TMP_LR) | RN(TMP_ZERO) | RM(src))); + else + FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_LR, src, srcw, TMP_REG1)); + + return push_inst(compiler, RET | RN(TMP_LR)); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + SLJIT_ASSERT(reg_map[1] == 0 && reg_map[3] == 2 && reg_map[5] == 4); + + /* The reg_map[op] should provide the appropriate constant. */ + if (op == SLJIT_PREFETCH_L1) + op = 1; + else if (op == SLJIT_PREFETCH_L2) + op = 3; + else if (op == SLJIT_PREFETCH_L3) + op = 5; + else + op = 2; + + /* Signed word sized load is the prefetch instruction. */ + return emit_op_mem(compiler, WORD_SIZE | SIGNED, op, src, srcw, TMP_REG1); + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -1529,20 +1596,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_LR, dst, dstw, TMP_REG1); } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst(compiler, ORR | RD(TMP_LR) | RN(TMP_ZERO) | RM(src))); - else - FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_LR, src, srcw, TMP_REG1)); - - return push_inst(compiler, RET | RN(TMP_LR)); -} - /* --------------------------------------------------------------------- */ /* Conditional instructions */ /* --------------------------------------------------------------------- */ @@ -1816,7 +1869,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compile CHECK_ERROR(); CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw)); - if ((mem & OFFS_REG_MASK) || (memw > 255 && memw < -256)) + if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256)) return SLJIT_ERR_UNSUPPORTED; if (type & SLJIT_MEM_SUPP) @@ -1866,7 +1919,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compil CHECK_ERROR(); CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw)); - if ((mem & OFFS_REG_MASK) || (memw > 255 && memw < -256)) + if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256)) return SLJIT_ERR_UNSUPPORTED; if (type & SLJIT_MEM_SUPP) @@ -1947,18 +2000,49 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi return const_; } +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1; + PTR_FAIL_IF(emit_imm64_const(compiler, dst_r, 0)); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 1); + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2)); + + return put_label; +} + SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins* inst = (sljit_ins*)addr; - modify_imm64_const(inst, new_target); + sljit_s32 dst; + SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0); + + dst = inst[0] & 0x1f; + SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21))); + inst[0] = MOVZ | dst | ((new_target & 0xffff) << 5); + inst[1] = MOVK | dst | (((new_target >> 16) & 0xffff) << 5) | (1 << 21); + inst[2] = MOVK | dst | (((new_target >> 32) & 0xffff) << 5) | (2 << 21); + inst[3] = MOVK | dst | ((new_target >> 48) << 5) | (3 << 21); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 4); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins* inst = (sljit_ins*)addr; - modify_imm64_const(inst, new_constant); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 4); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/dist2/src/sljit/sljitNativeARM_T2_32.c b/dist2/src/sljit/sljitNativeARM_T2_32.c index d7024b6d..4624882f 100644 --- a/dist2/src/sljit/sljitNativeARM_T2_32.c +++ b/dist2/src/sljit/sljitNativeARM_T2_32.c @@ -365,50 +365,64 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_u16 *buf_ptr; sljit_u16 *buf_end; sljit_uw half_count; + sljit_uw next_addr; sljit_sw executable_offset; struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_u16*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_u16)); + code = (sljit_u16*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_u16), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; code_ptr = code; half_count = 0; + next_addr = 0; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; do { buf_ptr = (sljit_u16*)buf->memory; buf_end = buf_ptr + (buf->used_size >> 1); do { *code_ptr = *buf_ptr++; - /* These structures are ordered by their address. */ - SLJIT_ASSERT(!label || label->size >= half_count); - SLJIT_ASSERT(!jump || jump->addr >= half_count); - SLJIT_ASSERT(!const_ || const_->addr >= half_count); - if (label && label->size == half_count) { - label->addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1; - label->size = code_ptr - code; - label = label->next; - } - if (jump && jump->addr == half_count) { - jump->addr = (sljit_uw)code_ptr - ((jump->flags & IS_COND) ? 10 : 8); - code_ptr -= detect_jump_type(jump, code_ptr, code, executable_offset); - jump = jump->next; - } - if (const_ && const_->addr == half_count) { - const_->addr = (sljit_uw)code_ptr; - const_ = const_->next; + if (next_addr == half_count) { + SLJIT_ASSERT(!label || label->size >= half_count); + SLJIT_ASSERT(!jump || jump->addr >= half_count); + SLJIT_ASSERT(!const_ || const_->addr >= half_count); + SLJIT_ASSERT(!put_label || put_label->addr >= half_count); + + /* These structures are ordered by their address. */ + if (label && label->size == half_count) { + label->addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1; + label->size = code_ptr - code; + label = label->next; + } + if (jump && jump->addr == half_count) { + jump->addr = (sljit_uw)code_ptr - ((jump->flags & IS_COND) ? 10 : 8); + code_ptr -= detect_jump_type(jump, code_ptr, code, executable_offset); + jump = jump->next; + } + if (const_ && const_->addr == half_count) { + const_->addr = (sljit_uw)code_ptr; + const_ = const_->next; + } + if (put_label && put_label->addr == half_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr ++; half_count ++; @@ -426,6 +440,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size); jump = compiler->jumps; @@ -434,6 +449,12 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } + put_label = compiler->put_labels; + while (put_label) { + modify_imm32_const((sljit_u16 *)put_label->addr, put_label->label->addr); + put_label = put_label->next; + } + compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = (code_ptr - code) * sizeof(sljit_u16); @@ -442,6 +463,8 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); + /* Set thumb mode flag. */ return (void*)((sljit_uw)code | 0x1); } @@ -459,6 +482,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) case SLJIT_HAS_CLZ: case SLJIT_HAS_CMOV: + case SLJIT_HAS_PREFETCH: return 1; default: @@ -586,7 +610,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s Although some clever things could be done here, "NOT IMM" does not worth the efforts. */ break; case SLJIT_ADD: - nimm = -imm; + nimm = -(sljit_sw)imm; if (IS_2_LO_REGS(reg, dst)) { if (imm <= 0x7) return push_inst16(compiler, ADDSI3 | IMM3(imm) | RD3(dst) | RN3(reg)); @@ -608,7 +632,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); break; @@ -633,11 +657,11 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, CMPI_W | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, CMNI_W | RN4(reg) | nimm); } - nimm = -imm; + nimm = -(sljit_sw)imm; if (IS_2_LO_REGS(reg, dst)) { if (imm <= 0x7) return push_inst16(compiler, SUBSI3 | IMM3(imm) | RD3(dst) | RN3(reg)); @@ -659,7 +683,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); break; @@ -1307,6 +1331,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile } return SLJIT_SUCCESS; #endif /* __ARM_FEATURE_IDIV || __ARM_ARCH_EXT_IDIV__ */ + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; @@ -1324,13 +1351,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { - /* Since TMP_PC has index 15, IS_2_LO_REGS and IS_3_LO_REGS checks always fail. */ - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) - return emit_op_mem(compiler, PRELOAD, TMP_PC, src, srcw, TMP_REG1); - return SLJIT_SUCCESS; - } - dst_r = SLOW_IS_REG(dst) ? dst : TMP_REG1; op = GET_OPCODE(op); @@ -1454,6 +1474,35 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return emit_op_mem(compiler, WORD_SIZE | STORE, dst_reg, dst, dstw, TMP_REG2); } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + SLJIT_ASSERT(reg_map[TMP_REG2] == 14); + + if (FAST_IS_REG(src)) + FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG2, src))); + else + FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2)); + + return push_inst16(compiler, BX | RN3(TMP_REG2)); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + return emit_op_mem(compiler, PRELOAD, TMP_PC, src, srcw, TMP_REG1); + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -1707,22 +1756,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, dst, dstw, TMP_REG1); } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - SLJIT_ASSERT(reg_map[TMP_REG2] == 14); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG2, src))); - else - FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2)); - - return push_inst16(compiler, BX | RN3(TMP_REG2)); -} - /* --------------------------------------------------------------------- */ /* Conditional instructions */ /* --------------------------------------------------------------------- */ @@ -2243,7 +2276,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compile CHECK_ERROR(); CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw)); - if ((mem & OFFS_REG_MASK) || (memw > 255 && memw < -255)) + if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -255)) return SLJIT_ERR_UNSUPPORTED; if (type & SLJIT_MEM_SUPP) @@ -2311,18 +2344,40 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi return const_; } +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1; + PTR_FAIL_IF(emit_imm32_const(compiler, dst_r, 0)); + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2)); + return put_label; +} + SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_u16 *inst = (sljit_u16*)addr; + SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0); modify_imm32_const(inst, new_target); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1); inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 4); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_u16 *inst = (sljit_u16*)addr; - modify_imm32_const(inst, new_constant); - inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 4); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/dist2/src/sljit/sljitNativeMIPS_32.c b/dist2/src/sljit/sljitNativeMIPS_32.c index ad970bf2..f887ee13 100644 --- a/dist2/src/sljit/sljitNativeMIPS_32.c +++ b/dist2/src/sljit/sljitNativeMIPS_32.c @@ -86,12 +86,12 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) { if (op == SLJIT_MOV_S8) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) return push_inst(compiler, SEB | T(src2) | D(dst), DR(dst)); -#else +#else /* SLJIT_MIPS_REV < 1 */ FAIL_IF(push_inst(compiler, SLL | T(src2) | D(dst) | SH_IMM(24), DR(dst))); return push_inst(compiler, SRA | T(dst) | D(dst) | SH_IMM(24), DR(dst)); -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ } return push_inst(compiler, ANDI | S(src2) | T(dst) | IMM(0xff), DR(dst)); } @@ -105,12 +105,12 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) { if (op == SLJIT_MOV_S16) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) return push_inst(compiler, SEH | T(src2) | D(dst), DR(dst)); -#else +#else /* SLJIT_MIPS_REV < 1 */ FAIL_IF(push_inst(compiler, SLL | T(src2) | D(dst) | SH_IMM(16), DR(dst))); return push_inst(compiler, SRA | T(dst) | D(dst) | SH_IMM(16), DR(dst)); -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ } return push_inst(compiler, ANDI | S(src2) | T(dst) | IMM(0xffff), DR(dst)); } @@ -129,12 +129,12 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl case SLJIT_CLZ: SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) if (op & SLJIT_SET_Z) FAIL_IF(push_inst(compiler, CLZ | S(src2) | TA(EQUAL_FLAG) | DA(EQUAL_FLAG), EQUAL_FLAG)); if (!(flags & UNUSED_DEST)) FAIL_IF(push_inst(compiler, CLZ | S(src2) | T(dst) | D(dst), DR(dst))); -#else +#else /* SLJIT_MIPS_REV < 1 */ if (SLJIT_UNLIKELY(flags & UNUSED_DEST)) { FAIL_IF(push_inst(compiler, SRL | T(src2) | DA(EQUAL_FLAG) | SH_IMM(31), EQUAL_FLAG)); return push_inst(compiler, XORI | SA(EQUAL_FLAG) | TA(EQUAL_FLAG) | IMM(1), EQUAL_FLAG); @@ -149,7 +149,7 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl FAIL_IF(push_inst(compiler, ADDIU | S(dst) | T(dst) | IMM(1), DR(dst))); FAIL_IF(push_inst(compiler, BGEZ | S(TMP_REG1) | IMM(-2), UNMOVABLE_INS)); FAIL_IF(push_inst(compiler, SLL | T(TMP_REG1) | D(TMP_REG1) | SH_IMM(1), UNMOVABLE_INS)); -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ return SLJIT_SUCCESS; case SLJIT_ADD: @@ -368,21 +368,22 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(!(flags & SRC2_IMM)); if (GET_FLAG_TYPE(op) != SLJIT_MUL_OVERFLOW) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) || (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) return push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst)); -#else /* !SLJIT_MIPS_R1 && !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 1 */ FAIL_IF(push_inst(compiler, MULT | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#endif /* SLJIT_MIPS_R1 || SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 1 */ } -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) FAIL_IF(push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst))); FAIL_IF(push_inst(compiler, MUH | S(src1) | T(src2) | DA(EQUAL_FLAG), EQUAL_FLAG)); -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ FAIL_IF(push_inst(compiler, MULT | S(src1) | T(src2), MOVABLE_INS)); FAIL_IF(push_inst(compiler, MFHI | DA(EQUAL_FLAG), EQUAL_FLAG)); FAIL_IF(push_inst(compiler, MFLO | D(dst), DR(dst))); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ FAIL_IF(push_inst(compiler, SRA | T(dst) | DA(OTHER_FLAG) | SH_IMM(31), OTHER_FLAG)); return push_inst(compiler, SUBU | SA(EQUAL_FLAG) | TA(OTHER_FLAG) | DA(OTHER_FLAG), OTHER_FLAG); @@ -424,21 +425,20 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + SLJIT_ASSERT((inst[0] & 0xffe00000) == LUI && (inst[1] & 0xfc000000) == ORI); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_ins *ins_ptr) diff --git a/dist2/src/sljit/sljitNativeMIPS_64.c b/dist2/src/sljit/sljitNativeMIPS_64.c index a6a2bcc0..5ab9b7d0 100644 --- a/dist2/src/sljit/sljitNativeMIPS_64.c +++ b/dist2/src/sljit/sljitNativeMIPS_64.c @@ -220,12 +220,12 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl case SLJIT_CLZ: SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) if (op & SLJIT_SET_Z) FAIL_IF(push_inst(compiler, SELECT_OP(DCLZ, CLZ) | S(src2) | TA(EQUAL_FLAG) | DA(EQUAL_FLAG), EQUAL_FLAG)); if (!(flags & UNUSED_DEST)) FAIL_IF(push_inst(compiler, SELECT_OP(DCLZ, CLZ) | S(src2) | T(dst) | D(dst), DR(dst))); -#else +#else /* SLJIT_MIPS_REV < 1 */ if (SLJIT_UNLIKELY(flags & UNUSED_DEST)) { FAIL_IF(push_inst(compiler, SELECT_OP(DSRL32, SRL) | T(src2) | DA(EQUAL_FLAG) | SH_IMM(31), EQUAL_FLAG)); return push_inst(compiler, XORI | SA(EQUAL_FLAG) | TA(EQUAL_FLAG) | IMM(1), EQUAL_FLAG); @@ -240,7 +240,7 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl FAIL_IF(push_inst(compiler, SELECT_OP(DADDIU, ADDIU) | S(dst) | T(dst) | IMM(1), DR(dst))); FAIL_IF(push_inst(compiler, BGEZ | S(TMP_REG1) | IMM(-2), UNMOVABLE_INS)); FAIL_IF(push_inst(compiler, SELECT_OP(DSLL, SLL) | T(TMP_REG1) | D(TMP_REG1) | SH_IMM(1), UNMOVABLE_INS)); -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ return SLJIT_SUCCESS; case SLJIT_ADD: @@ -459,26 +459,27 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(!(flags & SRC2_IMM)); if (GET_FLAG_TYPE(op) != SLJIT_MUL_OVERFLOW) { -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) return push_inst(compiler, SELECT_OP(DMUL, MUL) | S(src1) | T(src2) | D(dst), DR(dst)); -#elif (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#elif (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) if (op & SLJIT_I32_OP) return push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst)); FAIL_IF(push_inst(compiler, DMULT | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#else /* !SLJIT_MIPS_R6 && !SLJIT_MIPS_R1 */ +#else /* SLJIT_MIPS_REV < 1 */ FAIL_IF(push_inst(compiler, SELECT_OP(DMULT, MULT) | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ } -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) FAIL_IF(push_inst(compiler, SELECT_OP(DMUL, MUL) | S(src1) | T(src2) | D(dst), DR(dst))); FAIL_IF(push_inst(compiler, SELECT_OP(DMUH, MUH) | S(src1) | T(src2) | DA(EQUAL_FLAG), EQUAL_FLAG)); -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ FAIL_IF(push_inst(compiler, SELECT_OP(DMULT, MULT) | S(src1) | T(src2), MOVABLE_INS)); FAIL_IF(push_inst(compiler, MFHI | DA(EQUAL_FLAG), EQUAL_FLAG)); FAIL_IF(push_inst(compiler, MFLO | D(dst), DR(dst))); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ FAIL_IF(push_inst(compiler, SELECT_OP(DSRA32, SRA) | T(dst) | DA(OTHER_FLAG) | SH_IMM(31), OTHER_FLAG)); return push_inst(compiler, SELECT_OP(DSUBU, SUBU) | SA(EQUAL_FLAG) | TA(OTHER_FLAG) | DA(OTHER_FLAG), OTHER_FLAG); @@ -524,25 +525,21 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 6, 0); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 48) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | ((new_target >> 32) & 0xffff); inst[3] = (inst[3] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[5] = (inst[5] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 6, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 6); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 48) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | ((new_constant >> 32) & 0xffff); - inst[3] = (inst[3] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[5] = (inst[5] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 6); + sljit_set_jump_addr(addr, new_constant, executable_offset); } static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_ins *ins_ptr) diff --git a/dist2/src/sljit/sljitNativeMIPS_common.c b/dist2/src/sljit/sljitNativeMIPS_common.c index e0d6a3f0..ecf4dac4 100644 --- a/dist2/src/sljit/sljitNativeMIPS_common.c +++ b/dist2/src/sljit/sljitNativeMIPS_common.c @@ -25,15 +25,16 @@ */ /* Latest MIPS architecture. */ -/* Automatically detect SLJIT_MIPS_R1 */ -#if (defined __mips_isa_rev) && (__mips_isa_rev >= 6) -#define SLJIT_MIPS_R6 1 +#ifndef __mips_hard_float +/* Disable automatic detection, covers both -msoft-float and -mno-float */ +#undef SLJIT_IS_FPU_AVAILABLE +#define SLJIT_IS_FPU_AVAILABLE 0 #endif SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) { -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) return "MIPS32-R6" SLJIT_CPUINFO; @@ -41,7 +42,7 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) return "MIPS64-R6" SLJIT_CPUINFO; #endif /* SLJIT_CONFIG_MIPS_32 */ -#elif (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#elif (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) return "MIPS32-R1" SLJIT_CPUINFO; @@ -49,9 +50,9 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) return "MIPS64-R1" SLJIT_CPUINFO; #endif /* SLJIT_CONFIG_MIPS_32 */ -#else /* SLJIT_MIPS_R1 */ +#else /* SLJIT_MIPS_REV < 1 */ return "MIPS III" SLJIT_CPUINFO; -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ } /* Length of an instruction word @@ -117,11 +118,11 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define FR(dr) (freg_map[dr]) #define HI(opcode) ((opcode) << 26) #define LO(opcode) (opcode) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) /* CMP.cond.fmt */ /* S = (20 << 21) D = (21 << 21) */ #define CMP_FMT_S (20 << 21) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ /* S = (16 << 21) D = (17 << 21) */ #define FMT_S (16 << 21) #define FMT_D (17 << 21) @@ -134,13 +135,13 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define ANDI (HI(12)) #define B (HI(4)) #define BAL (HI(1) | (17 << 16)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define BC1EQZ (HI(17) | (9 << 21) | FT(TMP_FREG3)) #define BC1NEZ (HI(17) | (13 << 21) | FT(TMP_FREG3)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define BC1F (HI(17) | (8 << 21)) #define BC1T (HI(17) | (8 << 21) | (1 << 16)) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define BEQ (HI(4)) #define BGEZ (HI(1) | (1 << 16)) #define BGTZ (HI(7)) @@ -149,23 +150,23 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define BNE (HI(5)) #define BREAK (HI(0) | LO(13)) #define CFC1 (HI(17) | (2 << 21)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define C_UEQ_S (HI(17) | CMP_FMT_S | LO(3)) #define C_ULE_S (HI(17) | CMP_FMT_S | LO(7)) #define C_ULT_S (HI(17) | CMP_FMT_S | LO(5)) #define C_UN_S (HI(17) | CMP_FMT_S | LO(1)) #define C_FD (FD(TMP_FREG3)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define C_UEQ_S (HI(17) | FMT_S | LO(51)) #define C_ULE_S (HI(17) | FMT_S | LO(55)) #define C_ULT_S (HI(17) | FMT_S | LO(53)) #define C_UN_S (HI(17) | FMT_S | LO(49)) #define C_FD (0) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define CVT_S_S (HI(17) | FMT_S | LO(32)) #define DADDIU (HI(25)) #define DADDU (HI(0) | LO(45)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define DDIV (HI(0) | (2 << 6) | LO(30)) #define DDIVU (HI(0) | (2 << 6) | LO(31)) #define DMOD (HI(0) | (3 << 6) | LO(30)) @@ -176,14 +177,14 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define DMUHU (HI(0) | (3 << 6) | LO(29)) #define DMUL (HI(0) | (2 << 6) | LO(28)) #define DMULU (HI(0) | (2 << 6) | LO(29)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define DDIV (HI(0) | LO(30)) #define DDIVU (HI(0) | LO(31)) #define DIV (HI(0) | LO(26)) #define DIVU (HI(0) | LO(27)) #define DMULT (HI(0) | LO(28)) #define DMULTU (HI(0) | LO(29)) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define DIV_S (HI(17) | FMT_S | LO(3)) #define DSLL (HI(0) | LO(56)) #define DSLL32 (HI(0) | LO(60)) @@ -198,33 +199,33 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define J (HI(2)) #define JAL (HI(3)) #define JALR (HI(0) | LO(9)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define JR (HI(0) | LO(9)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define JR (HI(0) | LO(8)) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define LD (HI(55)) #define LUI (HI(15)) #define LW (HI(35)) #define MFC1 (HI(17)) -#if !(defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) -#define MFHI (HI(0) | LO(16)) -#define MFLO (HI(0) | LO(18)) -#else /* SLJIT_MIPS_R6 */ +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define MOD (HI(0) | (3 << 6) | LO(26)) #define MODU (HI(0) | (3 << 6) | LO(27)) -#endif /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ +#define MFHI (HI(0) | LO(16)) +#define MFLO (HI(0) | LO(18)) +#endif /* SLJIT_MIPS_REV >= 6 */ #define MOV_S (HI(17) | FMT_S | LO(6)) #define MTC1 (HI(17) | (4 << 21)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define MUH (HI(0) | (3 << 6) | LO(24)) #define MUHU (HI(0) | (3 << 6) | LO(25)) #define MUL (HI(0) | (2 << 6) | LO(24)) #define MULU (HI(0) | (2 << 6) | LO(25)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define MULT (HI(0) | LO(24)) #define MULTU (HI(0) | LO(25)) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define MUL_S (HI(17) | FMT_S | LO(2)) #define NEG_S (HI(17) | FMT_S | LO(7)) #define NOP (HI(0) | LO(0)) @@ -251,23 +252,23 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { #define XOR (HI(0) | LO(38)) #define XORI (HI(14)) -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) || (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) #define CLZ (HI(28) | LO(32)) -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define DCLZ (LO(18)) -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define DCLZ (HI(28) | LO(36)) #define MOVF (HI(0) | (0 << 16) | LO(1)) #define MOVN (HI(0) | LO(11)) #define MOVT (HI(0) | (1 << 16) | LO(1)) #define MOVZ (HI(0) | LO(10)) #define MUL (HI(28) | LO(2)) -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ #define PREF (HI(51)) #define PREFX (HI(19) | LO(15)) #define SEB (HI(31) | (16 << 6) | LO(32)) #define SEH (HI(31) | (24 << 6) | LO(32)) -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) #define ADDU_W ADDU @@ -289,9 +290,9 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { Useful for reordering instructions in the delay slot. */ static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins, sljit_s32 delay_slot) { + sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins)); SLJIT_ASSERT(delay_slot == MOVABLE_INS || delay_slot >= UNMOVABLE_INS || delay_slot == ((ins >> 11) & 0x1f) || delay_slot == ((ins >> 16) & 0x1f)); - sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins)); FAIL_IF(!ptr); *ptr = ins; compiler->size++; @@ -303,10 +304,10 @@ static SLJIT_INLINE sljit_ins invert_branch(sljit_s32 flags) { if (flags & IS_BIT26_COND) return (1 << 26); -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) if (flags & IS_BIT23_COND) return (1 << 23); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ return (1 << 16); } @@ -449,6 +450,55 @@ static __attribute__ ((noinline)) void sljit_cache_flush(void* code, void* code_ } #endif +#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) + +static SLJIT_INLINE sljit_sw put_label_get_length(struct sljit_put_label *put_label, sljit_uw max_label) +{ + if (max_label < 0x80000000l) { + put_label->flags = 0; + return 1; + } + + if (max_label < 0x800000000000l) { + put_label->flags = 1; + return 3; + } + + put_label->flags = 2; + return 5; +} + +static SLJIT_INLINE void put_label_set(struct sljit_put_label *put_label) +{ + sljit_uw addr = put_label->label->addr; + sljit_ins *inst = (sljit_ins *)put_label->addr; + sljit_s32 reg = *inst; + + if (put_label->flags == 0) { + SLJIT_ASSERT(addr < 0x80000000l); + inst[0] = LUI | T(reg) | IMM(addr >> 16); + } + else if (put_label->flags == 1) { + SLJIT_ASSERT(addr < 0x800000000000l); + inst[0] = LUI | T(reg) | IMM(addr >> 32); + inst[1] = ORI | S(reg) | T(reg) | IMM((addr >> 16) & 0xffff); + inst[2] = DSLL | T(reg) | D(reg) | SH_IMM(16); + inst += 2; + } + else { + inst[0] = LUI | T(reg) | IMM(addr >> 48); + inst[1] = ORI | S(reg) | T(reg) | IMM((addr >> 32) & 0xffff); + inst[2] = DSLL | T(reg) | D(reg) | SH_IMM(16); + inst[3] = ORI | S(reg) | T(reg) | IMM((addr >> 16) & 0xffff); + inst[4] = DSLL | T(reg) | D(reg) | SH_IMM(16); + inst += 4; + } + + inst[1] = ORI | S(reg) | T(reg) | IMM(addr & 0xffff); +} + +#endif + SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler) { struct sljit_memory_fragment *buf; @@ -457,56 +507,73 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_ins *buf_ptr; sljit_ins *buf_end; sljit_uw word_count; + sljit_uw next_addr; sljit_sw executable_offset; sljit_uw addr; struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; code_ptr = code; word_count = 0; + next_addr = 0; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; do { buf_ptr = (sljit_ins*)buf->memory; buf_end = buf_ptr + (buf->used_size >> 2); do { *code_ptr = *buf_ptr++; - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - /* These structures are ordered by their address. */ - if (label && label->size == word_count) { - label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); - label->size = code_ptr - code; - label = label->next; - } - if (jump && jump->addr == word_count) { + if (next_addr == word_count) { + SLJIT_ASSERT(!label || label->size >= word_count); + SLJIT_ASSERT(!jump || jump->addr >= word_count); + SLJIT_ASSERT(!const_ || const_->addr >= word_count); + SLJIT_ASSERT(!put_label || put_label->addr >= word_count); + + /* These structures are ordered by their address. */ + if (label && label->size == word_count) { + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + label->size = code_ptr - code; + label = label->next; + } + if (jump && jump->addr == word_count) { #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) - jump->addr = (sljit_uw)(code_ptr - 3); + jump->addr = (sljit_uw)(code_ptr - 3); #else - jump->addr = (sljit_uw)(code_ptr - 7); + jump->addr = (sljit_uw)(code_ptr - 7); #endif - code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset); - jump = jump->next; - } - if (const_ && const_->addr == word_count) { - /* Just recording the address. */ - const_->addr = (sljit_uw)code_ptr; - const_ = const_->next; + code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset); + jump = jump->next; + } + if (const_ && const_->addr == word_count) { + const_->addr = (sljit_uw)code_ptr; + const_ = const_->next; + } + if (put_label && put_label->addr == word_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; +#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) + code_ptr += put_label_get_length(put_label, (sljit_uw)(SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size)); + word_count += 5; +#endif + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr ++; word_count ++; @@ -524,6 +591,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size); jump = compiler->jumps; @@ -571,6 +639,21 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } + put_label = compiler->put_labels; + while (put_label) { +#if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) + addr = put_label->label->addr; + buf_ptr = (sljit_ins *)put_label->addr; + + SLJIT_ASSERT((buf_ptr[0] & 0xffe00000) == LUI && (buf_ptr[1] & 0xfc000000) == ORI); + buf_ptr[0] |= (addr >> 16) & 0xffff; + buf_ptr[1] |= addr & 0xffff; +#else + put_label_set(put_label); +#endif + put_label = put_label->next; + } + compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = (code_ptr - code) * sizeof(sljit_ins); @@ -584,6 +667,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil /* GCC workaround for invalid code generation with -O2. */ sljit_cache_flush(code, code_ptr); #endif + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -596,17 +680,20 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) #ifdef SLJIT_IS_FPU_AVAILABLE return SLJIT_IS_FPU_AVAILABLE; #elif defined(__GNUC__) - asm ("cfc1 %0, $0" : "=r"(fir)); + __asm__ ("cfc1 %0, $0" : "=r"(fir)); return (fir >> 22) & 0x1; #else #error "FIR check is not implemented for this architecture" #endif + case SLJIT_HAS_ZERO_REGISTER: + return 1; -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) case SLJIT_HAS_CLZ: case SLJIT_HAS_CMOV: + case SLJIT_HAS_PREFETCH: return 1; -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ default: return fir; @@ -1148,7 +1235,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile return push_inst(compiler, NOP, UNMOVABLE_INS); case SLJIT_LMUL_UW: case SLJIT_LMUL_SW: -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMULU : DMUL) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMUHU : DMUH) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); @@ -1158,7 +1245,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, ADDU_W | S(TMP_REG3) | TA(0) | D(SLJIT_R0), DR(SLJIT_R0))); return push_inst(compiler, ADDU_W | S(TMP_REG1) | TA(0) | D(SLJIT_R1), DR(SLJIT_R1)); -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMULTU : DMULT) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); #else /* !SLJIT_CONFIG_MIPS_64 */ @@ -1166,13 +1253,13 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, MFLO | D(SLJIT_R0), DR(SLJIT_R0))); return push_inst(compiler, MFHI | D(SLJIT_R1), DR(SLJIT_R1)); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ case SLJIT_DIVMOD_UW: case SLJIT_DIVMOD_SW: case SLJIT_DIV_UW: case SLJIT_DIV_SW: SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments); -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) if (int_op) { FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); @@ -1188,11 +1275,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, ADDU_W | S(TMP_REG3) | TA(0) | D(SLJIT_R0), DR(SLJIT_R0))); return (op >= SLJIT_DIV_UW) ? SLJIT_SUCCESS : push_inst(compiler, ADDU_W | S(TMP_REG1) | TA(0) | D(SLJIT_R1), DR(SLJIT_R1)); -#else /* !SLJIT_MIPS_R6 */ -#if !(defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#else /* SLJIT_MIPS_REV < 6 */ +#if !(defined SLJIT_MIPS_REV) FAIL_IF(push_inst(compiler, NOP, UNMOVABLE_INS)); FAIL_IF(push_inst(compiler, NOP, UNMOVABLE_INS)); -#endif /* !SLJIT_MIPS_R1 */ +#endif /* !SLJIT_MIPS_REV */ #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) if (int_op) FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); @@ -1203,13 +1290,16 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, MFLO | D(SLJIT_R0), DR(SLJIT_R0))); return (op >= SLJIT_DIV_UW) ? SLJIT_SUCCESS : push_inst(compiler, MFHI | D(SLJIT_R1), DR(SLJIT_R1)); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; } -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) { @@ -1230,7 +1320,7 @@ static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, return push_inst(compiler, PREFX | S(src & REG_MASK) | T(OFFS_REG(src)), MOVABLE_INS); } -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, @@ -1247,14 +1337,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) - return emit_prefetch(compiler, src, srcw); -#endif - return SLJIT_SUCCESS; - } - #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) if ((op & SLJIT_I32_OP) && GET_OPCODE(op) >= SLJIT_NOT) flags |= INT_DATA | SIGNED_DATA; @@ -1381,6 +1463,38 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile #endif } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + if (FAST_IS_REG(src)) + FAIL_IF(push_inst(compiler, ADDU_W | S(src) | TA(0) | DA(RETURN_ADDR_REG), RETURN_ADDR_REG)); + else + FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, RETURN_ADDR_REG, src, srcw)); + + FAIL_IF(push_inst(compiler, JR | SA(RETURN_ADDR_REG), UNMOVABLE_INS)); + return push_inst(compiler, NOP, UNMOVABLE_INS); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1) + return emit_prefetch(compiler, src, srcw); +#else /* SLJIT_MIPS_REV < 1 */ + return SLJIT_SUCCESS; +#endif /* SLJIT_MIPS_REV >= 1 */ + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -1650,25 +1764,12 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * ADJUST_LOCAL_OFFSET(dst, dstw); if (FAST_IS_REG(dst)) - return push_inst(compiler, ADDU_W | SA(RETURN_ADDR_REG) | TA(0) | D(dst), DR(dst)); + return push_inst(compiler, ADDU_W | SA(RETURN_ADDR_REG) | TA(0) | D(dst), UNMOVABLE_INS); /* Memory. */ - return emit_op_mem(compiler, WORD_DATA, RETURN_ADDR_REG, dst, dstw); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst(compiler, ADDU_W | S(src) | TA(0) | DA(RETURN_ADDR_REG), RETURN_ADDR_REG)); - else - FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, RETURN_ADDR_REG, src, srcw)); - - FAIL_IF(push_inst(compiler, JR | SA(RETURN_ADDR_REG), UNMOVABLE_INS)); - return push_inst(compiler, NOP, UNMOVABLE_INS); + FAIL_IF(emit_op_mem(compiler, WORD_DATA, RETURN_ADDR_REG, dst, dstw)); + compiler->delay_slot = UNMOVABLE_INS; + return SLJIT_SUCCESS; } /* --------------------------------------------------------------------- */ @@ -1708,7 +1809,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi flags = IS_BIT26_COND; \ delay_check = src; -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) #define BR_T() \ inst = BC1NEZ; \ @@ -1719,7 +1820,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi flags = IS_BIT23_COND; \ delay_check = FCSR_FCC; -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ #define BR_T() \ inst = BC1T | JUMP_LENGTH; \ @@ -1730,7 +1831,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi flags = IS_BIT16_COND; \ delay_check = FCSR_FCC; -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type) { @@ -2041,11 +2142,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *co case SLJIT_GREATER_EQUAL_F64: case SLJIT_UNORDERED_F64: case SLJIT_ORDERED_F64: -#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6) FAIL_IF(push_inst(compiler, MFC1 | TA(dst_ar) | FS(TMP_FREG3), dst_ar)); -#else /* !SLJIT_MIPS_R6 */ +#else /* SLJIT_MIPS_REV < 6 */ FAIL_IF(push_inst(compiler, CFC1 | TA(dst_ar) | DA(FCSR_REG), dst_ar)); -#endif /* SLJIT_MIPS_R6 */ +#endif /* SLJIT_MIPS_REV >= 6 */ FAIL_IF(push_inst(compiler, SRL | TA(dst_ar) | DA(dst_ar) | SH_IMM(23), dst_ar)); FAIL_IF(push_inst(compiler, ANDI | SA(dst_ar) | TA(dst_ar) | IMM(1), dst_ar)); src_ar = dst_ar; @@ -2085,14 +2186,14 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compil sljit_s32 dst_reg, sljit_s32 src, sljit_sw srcw) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1 && SLJIT_MIPS_REV < 6) sljit_ins ins; -#endif +#endif /* SLJIT_MIPS_REV >= 1 && SLJIT_MIPS_REV < 6 */ CHECK_ERROR(); CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw)); -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 1 && SLJIT_MIPS_REV < 6) if (SLJIT_UNLIKELY(src & SLJIT_IMM)) { #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) @@ -2149,15 +2250,15 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compil return push_inst(compiler, ins | S(src) | D(dst_reg), DR(dst_reg)); -#else +#else /* SLJIT_MIPS_REV < 1 || SLJIT_MIPS_REV >= 6 */ return sljit_emit_cmov_generic(compiler, type, dst_reg, src, srcw); -#endif +#endif /* SLJIT_MIPS_REV >= 1 */ } SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) { struct sljit_const *const_; - sljit_s32 reg; + sljit_s32 dst_r; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); @@ -2167,11 +2268,38 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi PTR_FAIL_IF(!const_); set_const(const_, compiler); - reg = FAST_IS_REG(dst) ? dst : TMP_REG2; - - PTR_FAIL_IF(emit_const(compiler, reg, init_value)); + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; + PTR_FAIL_IF(emit_const(compiler, dst_r, init_value)); if (dst & SLJIT_MEM) PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0)); + return const_; } + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; +#if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) + PTR_FAIL_IF(emit_const(compiler, dst_r, 0)); +#else + PTR_FAIL_IF(push_inst(compiler, dst_r, UNMOVABLE_INS)); + compiler->size += 5; +#endif + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0)); + + return put_label; +} diff --git a/dist2/src/sljit/sljitNativePPC_32.c b/dist2/src/sljit/sljitNativePPC_32.c index fc185f78..7d9ec533 100644 --- a/dist2/src/sljit/sljitNativePPC_32.c +++ b/dist2/src/sljit/sljitNativePPC_32.c @@ -258,19 +258,18 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + SLJIT_ASSERT((inst[0] & 0xfc1f0000) == ADDIS && (inst[1] & 0xfc000000) == ORI); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/dist2/src/sljit/sljitNativePPC_64.c b/dist2/src/sljit/sljitNativePPC_64.c index 706b2ba2..92147d2a 100644 --- a/dist2/src/sljit/sljitNativePPC_64.c +++ b/dist2/src/sljit/sljitNativePPC_64.c @@ -35,9 +35,6 @@ #error "Must implement count leading zeroes" #endif -#define RLDI(dst, src, sh, mb, type) \ - (HI(30) | S(src) | A(dst) | ((type) << 2) | (((sh) & 0x1f) << 11) | (((sh) & 0x20) >> 4) | (((mb) & 0x1f) << 6) | ((mb) & 0x20)) - #define PUSH_RLDICR(reg, shift) \ push_inst(compiler, RLDI(reg, reg, 63 - shift, shift, 1)) @@ -480,23 +477,19 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins*)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 5, 0); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 48) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | ((new_target >> 32) & 0xffff); inst[3] = (inst[3] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[4] = (inst[4] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 5, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 5); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins*)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 48) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | ((new_constant >> 32) & 0xffff); - inst[3] = (inst[3] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[4] = (inst[4] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 5); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/dist2/src/sljit/sljitNativePPC_common.c b/dist2/src/sljit/sljitNativePPC_common.c index b34e3965..d84562ce 100644 --- a/dist2/src/sljit/sljitNativePPC_common.c +++ b/dist2/src/sljit/sljitNativePPC_common.c @@ -231,6 +231,9 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define SIMM_MIN (-0x8000) #define UIMM_MAX (0xffff) +#define RLDI(dst, src, sh, mb, type) \ + (HI(30) | S(src) | A(dst) | ((type) << 2) | (((sh) & 0x1f) << 11) | (((sh) & 0x20) >> 4) | (((mb) & 0x1f) << 6) | ((mb) & 0x20)) + #if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) SLJIT_API_FUNC_ATTRIBUTE void sljit_set_function_context(void** func_ptr, struct sljit_function_context* context, sljit_sw addr, void* func) { @@ -324,6 +327,55 @@ keep_address: return 0; } +#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) + +static SLJIT_INLINE sljit_sw put_label_get_length(struct sljit_put_label *put_label, sljit_uw max_label) +{ + if (max_label < 0x100000000l) { + put_label->flags = 0; + return 1; + } + + if (max_label < 0x1000000000000l) { + put_label->flags = 1; + return 3; + } + + put_label->flags = 2; + return 4; +} + +static SLJIT_INLINE void put_label_set(struct sljit_put_label *put_label) +{ + sljit_uw addr = put_label->label->addr; + sljit_ins *inst = (sljit_ins *)put_label->addr; + sljit_s32 reg = *inst; + + if (put_label->flags == 0) { + SLJIT_ASSERT(addr < 0x100000000l); + inst[0] = ORIS | S(TMP_ZERO) | A(reg) | IMM(addr >> 16); + } + else { + if (put_label->flags == 1) { + SLJIT_ASSERT(addr < 0x1000000000000l); + inst[0] = ORI | S(TMP_ZERO) | A(reg) | IMM(addr >> 32); + } + else { + inst[0] = ORIS | S(TMP_ZERO) | A(reg) | IMM(addr >> 48); + inst[1] = ORI | S(reg) | A(reg) | IMM((addr >> 32) & 0xffff); + inst ++; + } + + inst[1] = RLDI(reg, reg, 32, 31, 1); + inst[2] = ORIS | S(reg) | A(reg) | IMM((addr >> 16) & 0xffff); + inst += 2; + } + + inst[1] = ORI | S(reg) | A(reg) | IMM(addr & 0xffff); +} + +#endif + SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler) { struct sljit_memory_fragment *buf; @@ -332,12 +384,14 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_ins *buf_ptr; sljit_ins *buf_end; sljit_uw word_count; + sljit_uw next_addr; sljit_sw executable_offset; sljit_uw addr; struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); @@ -350,77 +404,93 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil compiler->size += (sizeof(struct sljit_function_context) / sizeof(sljit_ins)); #endif #endif - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; code_ptr = code; word_count = 0; + next_addr = 0; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; do { buf_ptr = (sljit_ins*)buf->memory; buf_end = buf_ptr + (buf->used_size >> 2); do { *code_ptr = *buf_ptr++; - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - /* These structures are ordered by their address. */ - if (label && label->size == word_count) { - /* Just recording the address. */ - label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); - label->size = code_ptr - code; - label = label->next; - } - if (jump && jump->addr == word_count) { + if (next_addr == word_count) { + SLJIT_ASSERT(!label || label->size >= word_count); + SLJIT_ASSERT(!jump || jump->addr >= word_count); + SLJIT_ASSERT(!const_ || const_->addr >= word_count); + SLJIT_ASSERT(!put_label || put_label->addr >= word_count); + + /* These structures are ordered by their address. */ + if (label && label->size == word_count) { + /* Just recording the address. */ + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + label->size = code_ptr - code; + label = label->next; + } + if (jump && jump->addr == word_count) { #if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) - jump->addr = (sljit_uw)(code_ptr - 3); + jump->addr = (sljit_uw)(code_ptr - 3); #else - jump->addr = (sljit_uw)(code_ptr - 6); + jump->addr = (sljit_uw)(code_ptr - 6); #endif - if (detect_jump_type(jump, code_ptr, code, executable_offset)) { + if (detect_jump_type(jump, code_ptr, code, executable_offset)) { #if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) - code_ptr[-3] = code_ptr[0]; - code_ptr -= 3; -#else - if (jump->flags & PATCH_ABS32) { + code_ptr[-3] = code_ptr[0]; code_ptr -= 3; - code_ptr[-1] = code_ptr[2]; - code_ptr[0] = code_ptr[3]; - } - else if (jump->flags & PATCH_ABS48) { - code_ptr--; - code_ptr[-1] = code_ptr[0]; - code_ptr[0] = code_ptr[1]; - /* rldicr rX,rX,32,31 -> rX,rX,16,47 */ - SLJIT_ASSERT((code_ptr[-3] & 0xfc00ffff) == 0x780007c6); - code_ptr[-3] ^= 0x8422; - /* oris -> ori */ - code_ptr[-2] ^= 0x4000000; - } - else { - code_ptr[-6] = code_ptr[0]; - code_ptr -= 6; - } +#else + if (jump->flags & PATCH_ABS32) { + code_ptr -= 3; + code_ptr[-1] = code_ptr[2]; + code_ptr[0] = code_ptr[3]; + } + else if (jump->flags & PATCH_ABS48) { + code_ptr--; + code_ptr[-1] = code_ptr[0]; + code_ptr[0] = code_ptr[1]; + /* rldicr rX,rX,32,31 -> rX,rX,16,47 */ + SLJIT_ASSERT((code_ptr[-3] & 0xfc00ffff) == 0x780007c6); + code_ptr[-3] ^= 0x8422; + /* oris -> ori */ + code_ptr[-2] ^= 0x4000000; + } + else { + code_ptr[-6] = code_ptr[0]; + code_ptr -= 6; + } #endif - if (jump->flags & REMOVE_COND) { - code_ptr[0] = BCx | (2 << 2) | ((code_ptr[0] ^ (8 << 21)) & 0x03ff0001); - code_ptr++; - jump->addr += sizeof(sljit_ins); - code_ptr[0] = Bx; - jump->flags -= IS_COND; + if (jump->flags & REMOVE_COND) { + code_ptr[0] = BCx | (2 << 2) | ((code_ptr[0] ^ (8 << 21)) & 0x03ff0001); + code_ptr++; + jump->addr += sizeof(sljit_ins); + code_ptr[0] = Bx; + jump->flags -= IS_COND; + } } + jump = jump->next; } - jump = jump->next; - } - if (const_ && const_->addr == word_count) { - const_->addr = (sljit_uw)code_ptr; - const_ = const_->next; + if (const_ && const_->addr == word_count) { + const_->addr = (sljit_uw)code_ptr; + const_ = const_->next; + } + if (put_label && put_label->addr == word_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; +#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64) + code_ptr += put_label_get_length(put_label, (sljit_uw)(SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size)); + word_count += 4; +#endif + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr ++; word_count ++; @@ -438,6 +508,8 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); + #if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size - (sizeof(struct sljit_function_context) / sizeof(sljit_ins))); #else @@ -503,6 +575,21 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } + put_label = compiler->put_labels; + while (put_label) { +#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) + addr = put_label->label->addr; + buf_ptr = (sljit_ins *)put_label->addr; + + SLJIT_ASSERT((buf_ptr[0] & 0xfc1f0000) == ADDIS && (buf_ptr[1] & 0xfc000000) == ORI); + buf_ptr[0] |= (addr >> 16) & 0xffff; + buf_ptr[1] |= addr & 0xffff; +#else + put_label_set(put_label); +#endif + put_label = put_label->next; + } + compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = (code_ptr - code) * sizeof(sljit_ins); @@ -520,6 +607,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); #if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) return code_ptr; @@ -539,7 +627,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) return 1; #endif + /* A saved register is set to a zero value. */ + case SLJIT_HAS_ZERO_REGISTER: case SLJIT_HAS_CLZ: + case SLJIT_HAS_PREFETCH: return 1; default: @@ -1071,6 +1162,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #else return push_inst(compiler, (op == SLJIT_DIV_UW ? DIVWU : DIVW) | D(SLJIT_R0) | A(SLJIT_R0) | B(SLJIT_R1)); #endif + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; @@ -1116,13 +1210,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) - return emit_prefetch(compiler, src, srcw); - - return SLJIT_SUCCESS; - } - op = GET_OPCODE(op); if ((src & SLJIT_IMM) && srcw == 0) src = TMP_ZERO; @@ -1449,6 +1536,35 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_SUCCESS; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + if (FAST_IS_REG(src)) + FAIL_IF(push_inst(compiler, MTLR | S(src))); + else { + FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, TMP_REG2, 0, TMP_REG1, 0, src, srcw)); + FAIL_IF(push_inst(compiler, MTLR | S(TMP_REG2))); + } + + return push_inst(compiler, BLR); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + return emit_prefetch(compiler, src, srcw); + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -1767,22 +1883,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0); } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst(compiler, MTLR | S(src))); - else { - FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, TMP_REG2, 0, TMP_REG1, 0, src, srcw)); - FAIL_IF(push_inst(compiler, MTLR | S(TMP_REG2))); - } - - return push_inst(compiler, BLR); -} - /* --------------------------------------------------------------------- */ /* Conditional instructions */ /* --------------------------------------------------------------------- */ @@ -2261,7 +2361,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compil SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) { struct sljit_const *const_; - sljit_s32 reg; + sljit_s32 dst_r; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); @@ -2271,11 +2371,38 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi PTR_FAIL_IF(!const_); set_const(const_, compiler); - reg = FAST_IS_REG(dst) ? dst : TMP_REG2; - - PTR_FAIL_IF(emit_const(compiler, reg, init_value)); + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; + PTR_FAIL_IF(emit_const(compiler, dst_r, init_value)); if (dst & SLJIT_MEM) PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0)); + return const_; } + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; +#if (defined SLJIT_CONFIG_PPC_32 && SLJIT_CONFIG_PPC_32) + PTR_FAIL_IF(emit_const(compiler, dst_r, 0)); +#else + PTR_FAIL_IF(push_inst(compiler, dst_r)); + compiler->size += 4; +#endif + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0)); + + return put_label; +} diff --git a/dist2/src/sljit/sljitNativeS390X.c b/dist2/src/sljit/sljitNativeS390X.c new file mode 100644 index 00000000..a8b65112 --- /dev/null +++ b/dist2/src/sljit/sljitNativeS390X.c @@ -0,0 +1,2812 @@ +/* + * Stack-less Just-In-Time compiler + * + * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/auxv.h> + +#ifdef __ARCH__ +#define ENABLE_STATIC_FACILITY_DETECTION 1 +#else +#define ENABLE_STATIC_FACILITY_DETECTION 0 +#endif +#define ENABLE_DYNAMIC_FACILITY_DETECTION 1 + +SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) +{ + return "s390x" SLJIT_CPUINFO; +} + +/* Instructions. */ +typedef sljit_uw sljit_ins; + +/* Instruction tags (most significant halfword). */ +const sljit_ins sljit_ins_const = (sljit_ins)1 << 48; + +static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 4] = { + 14, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 0, 1 +}; + +/* there are also a[2-15] available, but they are slower to access and + * their use is limited as mundaym explained: + * https://github.com/zherczeg/sljit/pull/91#discussion_r486895689 + */ + +/* General Purpose Registers [0-15]. */ +typedef sljit_uw sljit_gpr; + +/* + * WARNING + * the following code is non standard and should be improved for + * consistency, but doesn't use SLJIT_NUMBER_OF_REGISTERS based + * registers because r0 and r1 are the ABI recommended volatiles. + * there is a gpr() function that maps sljit to physical register numbers + * that should be used instead of the usual index into reg_map[] and + * will be retired ASAP (TODO: carenas) + */ + +const sljit_gpr r0 = 0; /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 2]: 0 in address calculations; reserved */ +const sljit_gpr r1 = 1; /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 3]: reserved */ +const sljit_gpr r2 = 2; /* reg_map[1]: 1st argument */ +const sljit_gpr r3 = 3; /* reg_map[2]: 2nd argument */ +const sljit_gpr r4 = 4; /* reg_map[3]: 3rd argument */ +const sljit_gpr r5 = 5; /* reg_map[4]: 4th argument */ +const sljit_gpr r6 = 6; /* reg_map[5]: 5th argument; 1st saved register */ +const sljit_gpr r7 = 7; /* reg_map[6] */ +const sljit_gpr r8 = 8; /* reg_map[7] */ +const sljit_gpr r9 = 9; /* reg_map[8] */ +const sljit_gpr r10 = 10; /* reg_map[9] */ +const sljit_gpr r11 = 11; /* reg_map[10] */ +const sljit_gpr r12 = 12; /* reg_map[11]: GOT */ +const sljit_gpr r13 = 13; /* reg_map[12]: Literal Pool pointer */ +const sljit_gpr r14 = 14; /* reg_map[0]: return address and flag register */ +const sljit_gpr r15 = 15; /* reg_map[SLJIT_NUMBER_OF_REGISTERS + 1]: stack pointer */ + +/* WARNING: r12 and r13 shouldn't be used as per ABI recommendation */ +/* TODO(carenas): r12 might conflict in PIC code, reserve? */ +/* TODO(carenas): r13 is usually pointed to "pool" per ABI, using a tmp + * like we do know might be faster though, reserve? + */ + +/* TODO(carenas): should be named TMP_REG[1-2] for consistency */ +#define tmp0 r0 +#define tmp1 r1 + +/* TODO(carenas): flags should move to a different register so that + * link register doesn't need to change + */ + +/* Link registers. The normal link register is r14, but since + we use that for flags we need to use r0 instead to do fast + calls so that flags are preserved. */ +const sljit_gpr link_r = 14; /* r14 */ +const sljit_gpr fast_link_r = 0; /* r0 */ + +/* Flag register layout: + + 0 32 33 34 36 64 + +---------------+---+---+-------+-------+ + | ZERO | 0 | 0 | C C |///////| + +---------------+---+---+-------+-------+ +*/ +const sljit_gpr flag_r = 14; /* r14 */ + +struct sljit_s390x_const { + struct sljit_const const_; /* must be first */ + sljit_sw init_value; /* required to build literal pool */ +}; + +/* Convert SLJIT register to hardware register. */ +static SLJIT_INLINE sljit_gpr gpr(sljit_s32 r) +{ + SLJIT_ASSERT(r != SLJIT_UNUSED); + SLJIT_ASSERT(r < (sljit_s32)(sizeof(reg_map) / sizeof(reg_map[0]))); + return reg_map[r]; +} + +/* Size of instruction in bytes. Tags must already be cleared. */ +static SLJIT_INLINE sljit_uw sizeof_ins(sljit_ins ins) +{ + /* keep faulting instructions */ + if (ins == 0) + return 2; + + if ((ins & 0x00000000ffffL) == ins) + return 2; + if ((ins & 0x0000ffffffffL) == ins) + return 4; + if ((ins & 0xffffffffffffL) == ins) + return 6; + + SLJIT_UNREACHABLE(); + return (sljit_uw)-1; +} + +static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins) +{ + sljit_ins *ibuf = (sljit_ins *)ensure_buf(compiler, sizeof(sljit_ins)); + FAIL_IF(!ibuf); + *ibuf = ins; + compiler->size++; + return SLJIT_SUCCESS; +} + +static sljit_s32 encode_inst(void **ptr, sljit_ins ins) +{ + sljit_u16 *ibuf = (sljit_u16 *)*ptr; + sljit_uw size = sizeof_ins(ins); + + SLJIT_ASSERT((size & 6) == size); + switch (size) { + case 6: + *ibuf++ = (sljit_u16)(ins >> 32); + /* fallthrough */ + case 4: + *ibuf++ = (sljit_u16)(ins >> 16); + /* fallthrough */ + case 2: + *ibuf++ = (sljit_u16)(ins); + } + *ptr = (void*)ibuf; + return SLJIT_SUCCESS; +} + +/* Map the given type to a 4-bit condition code mask. */ +static SLJIT_INLINE sljit_u8 get_cc(sljit_s32 type) { + const sljit_u8 eq = 1 << 3; /* equal {,to zero} */ + const sljit_u8 lt = 1 << 2; /* less than {,zero} */ + const sljit_u8 gt = 1 << 1; /* greater than {,zero} */ + const sljit_u8 ov = 1 << 0; /* {overflow,NaN} */ + + switch (type) { + case SLJIT_EQUAL: + case SLJIT_EQUAL_F64: + return eq; + + case SLJIT_NOT_EQUAL: + case SLJIT_NOT_EQUAL_F64: + return ~eq; + + case SLJIT_LESS: + case SLJIT_SIG_LESS: + case SLJIT_LESS_F64: + return lt; + + case SLJIT_LESS_EQUAL: + case SLJIT_SIG_LESS_EQUAL: + case SLJIT_LESS_EQUAL_F64: + return (lt | eq); + + case SLJIT_GREATER: + case SLJIT_SIG_GREATER: + case SLJIT_GREATER_F64: + return gt; + + case SLJIT_GREATER_EQUAL: + case SLJIT_SIG_GREATER_EQUAL: + case SLJIT_GREATER_EQUAL_F64: + return (gt | eq); + + case SLJIT_OVERFLOW: + case SLJIT_MUL_OVERFLOW: + case SLJIT_UNORDERED_F64: + return ov; + + case SLJIT_NOT_OVERFLOW: + case SLJIT_MUL_NOT_OVERFLOW: + case SLJIT_ORDERED_F64: + return ~ov; + } + + SLJIT_UNREACHABLE(); + return (sljit_u8)-1; +} + +/* Facility to bit index mappings. + Note: some facilities share the same bit index. */ +typedef sljit_uw facility_bit; +#define STORE_FACILITY_LIST_EXTENDED_FACILITY 7 +#define FAST_LONG_DISPLACEMENT_FACILITY 19 +#define EXTENDED_IMMEDIATE_FACILITY 21 +#define GENERAL_INSTRUCTION_EXTENSION_FACILITY 34 +#define DISTINCT_OPERAND_FACILITY 45 +#define HIGH_WORD_FACILITY 45 +#define POPULATION_COUNT_FACILITY 45 +#define LOAD_STORE_ON_CONDITION_1_FACILITY 45 +#define MISCELLANEOUS_INSTRUCTION_EXTENSIONS_1_FACILITY 49 +#define LOAD_STORE_ON_CONDITION_2_FACILITY 53 +#define MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY 58 +#define VECTOR_FACILITY 129 +#define VECTOR_ENHANCEMENTS_1_FACILITY 135 + +/* Report whether a facility is known to be present due to the compiler + settings. This function should always be compiled to a constant + value given a constant argument. */ +static SLJIT_INLINE int have_facility_static(facility_bit x) +{ +#if ENABLE_STATIC_FACILITY_DETECTION + switch (x) { + case FAST_LONG_DISPLACEMENT_FACILITY: + return (__ARCH__ >= 6 /* z990 */); + case EXTENDED_IMMEDIATE_FACILITY: + case STORE_FACILITY_LIST_EXTENDED_FACILITY: + return (__ARCH__ >= 7 /* z9-109 */); + case GENERAL_INSTRUCTION_EXTENSION_FACILITY: + return (__ARCH__ >= 8 /* z10 */); + case DISTINCT_OPERAND_FACILITY: + return (__ARCH__ >= 9 /* z196 */); + case MISCELLANEOUS_INSTRUCTION_EXTENSIONS_1_FACILITY: + return (__ARCH__ >= 10 /* zEC12 */); + case LOAD_STORE_ON_CONDITION_2_FACILITY: + case VECTOR_FACILITY: + return (__ARCH__ >= 11 /* z13 */); + case MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY: + case VECTOR_ENHANCEMENTS_1_FACILITY: + return (__ARCH__ >= 12 /* z14 */); + default: + SLJIT_UNREACHABLE(); + } +#endif + return 0; +} + +static SLJIT_INLINE unsigned long get_hwcap() +{ + static unsigned long hwcap = 0; + if (SLJIT_UNLIKELY(!hwcap)) { + hwcap = getauxval(AT_HWCAP); + SLJIT_ASSERT(hwcap != 0); + } + return hwcap; +} + +static SLJIT_INLINE int have_stfle() +{ + if (have_facility_static(STORE_FACILITY_LIST_EXTENDED_FACILITY)) + return 1; + + return (get_hwcap() & HWCAP_S390_STFLE); +} + +/* Report whether the given facility is available. This function always + performs a runtime check. */ +static int have_facility_dynamic(facility_bit x) +{ +#if ENABLE_DYNAMIC_FACILITY_DETECTION + static struct { + sljit_uw bits[4]; + } cpu_features; + size_t size = sizeof(cpu_features); + const sljit_uw word_index = x >> 6; + const sljit_uw bit_index = ((1UL << 63) >> (x & 63)); + + SLJIT_ASSERT(x < size * 8); + if (SLJIT_UNLIKELY(!have_stfle())) + return 0; + + if (SLJIT_UNLIKELY(cpu_features.bits[0] == 0)) { + __asm__ __volatile__ ( + "lgr %%r0, %0;" + "stfle 0(%1);" + /* outputs */: + /* inputs */: "d" ((size / 8) - 1), "a" (&cpu_features) + /* clobbers */: "r0", "cc", "memory" + ); + SLJIT_ASSERT(cpu_features.bits[0] != 0); + } + return (cpu_features.bits[word_index] & bit_index) != 0; +#else + return 0; +#endif +} + +#define HAVE_FACILITY(name, bit) \ +static SLJIT_INLINE int name() \ +{ \ + static int have = -1; \ + /* Static check first. May allow the function to be optimized away. */ \ + if (have_facility_static(bit)) \ + have = 1; \ + else if (SLJIT_UNLIKELY(have < 0)) \ + have = have_facility_dynamic(bit) ? 1 : 0; \ +\ + return have; \ +} + +HAVE_FACILITY(have_eimm, EXTENDED_IMMEDIATE_FACILITY) +HAVE_FACILITY(have_ldisp, FAST_LONG_DISPLACEMENT_FACILITY) +HAVE_FACILITY(have_genext, GENERAL_INSTRUCTION_EXTENSION_FACILITY) +HAVE_FACILITY(have_lscond1, LOAD_STORE_ON_CONDITION_1_FACILITY) +HAVE_FACILITY(have_lscond2, LOAD_STORE_ON_CONDITION_2_FACILITY) +HAVE_FACILITY(have_misc2, MISCELLANEOUS_INSTRUCTION_EXTENSIONS_2_FACILITY) +#undef HAVE_FACILITY + +#define is_u12(d) (0 <= (d) && (d) <= 0x00000fffL) +#define is_u32(d) (0 <= (d) && (d) <= 0xffffffffL) + +#define CHECK_SIGNED(v, bitlen) \ + ((v) == (((v) << (sizeof(v) * 8 - bitlen)) >> (sizeof(v) * 8 - bitlen))) + +#define is_s16(d) CHECK_SIGNED((d), 16) +#define is_s20(d) CHECK_SIGNED((d), 20) +#define is_s32(d) CHECK_SIGNED((d), 32) + +static SLJIT_INLINE sljit_uw disp_s20(sljit_s32 d) +{ + sljit_uw dh = (d >> 12) & 0xff; + sljit_uw dl = (d << 8) & 0xfff00; + + SLJIT_ASSERT(is_s20(d)); + return dh | dl; +} + +/* TODO(carenas): variadic macro is not strictly needed */ +#define SLJIT_S390X_INSTRUCTION(op, ...) \ +static SLJIT_INLINE sljit_ins op(__VA_ARGS__) + +/* RR form instructions. */ +#define SLJIT_S390X_RR(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src) \ +{ \ + return (pattern) | ((dst & 0xf) << 4) | (src & 0xf); \ +} + +/* ADD */ +SLJIT_S390X_RR(ar, 0x1a00) + +/* ADD LOGICAL */ +SLJIT_S390X_RR(alr, 0x1e00) + +/* AND */ +SLJIT_S390X_RR(nr, 0x1400) + +/* BRANCH AND SAVE */ +SLJIT_S390X_RR(basr, 0x0d00) + +/* BRANCH ON CONDITION */ +SLJIT_S390X_RR(bcr, 0x0700) /* TODO(mundaym): type for mask? */ + +/* COMPARE */ +SLJIT_S390X_RR(cr, 0x1900) + +/* COMPARE LOGICAL */ +SLJIT_S390X_RR(clr, 0x1500) + +/* DIVIDE */ +SLJIT_S390X_RR(dr, 0x1d00) + +/* EXCLUSIVE OR */ +SLJIT_S390X_RR(xr, 0x1700) + +/* LOAD */ +SLJIT_S390X_RR(lr, 0x1800) + +/* LOAD COMPLEMENT */ +SLJIT_S390X_RR(lcr, 0x1300) + +/* OR */ +SLJIT_S390X_RR(or, 0x1600) + +/* SUBTRACT */ +SLJIT_S390X_RR(sr, 0x1b00) + +/* SUBTRACT LOGICAL */ +SLJIT_S390X_RR(slr, 0x1f00) + +#undef SLJIT_S390X_RR + +/* RRE form instructions */ +#define SLJIT_S390X_RRE(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src) \ +{ \ + return (pattern) | ((dst & 0xf) << 4) | (src & 0xf); \ +} + +/* ADD */ +SLJIT_S390X_RRE(agr, 0xb9080000) + +/* ADD LOGICAL */ +SLJIT_S390X_RRE(algr, 0xb90a0000) + +/* ADD LOGICAL WITH CARRY */ +SLJIT_S390X_RRE(alcr, 0xb9980000) +SLJIT_S390X_RRE(alcgr, 0xb9880000) + +/* AND */ +SLJIT_S390X_RRE(ngr, 0xb9800000) + +/* COMPARE */ +SLJIT_S390X_RRE(cgr, 0xb9200000) + +/* COMPARE LOGICAL */ +SLJIT_S390X_RRE(clgr, 0xb9210000) + +/* DIVIDE LOGICAL */ +SLJIT_S390X_RRE(dlr, 0xb9970000) +SLJIT_S390X_RRE(dlgr, 0xb9870000) + +/* DIVIDE SINGLE */ +SLJIT_S390X_RRE(dsgr, 0xb90d0000) + +/* EXCLUSIVE OR */ +SLJIT_S390X_RRE(xgr, 0xb9820000) + +/* LOAD */ +SLJIT_S390X_RRE(lgr, 0xb9040000) +SLJIT_S390X_RRE(lgfr, 0xb9140000) + +/* LOAD BYTE */ +SLJIT_S390X_RRE(lbr, 0xb9260000) +SLJIT_S390X_RRE(lgbr, 0xb9060000) + +/* LOAD COMPLEMENT */ +SLJIT_S390X_RRE(lcgr, 0xb9030000) + +/* LOAD HALFWORD */ +SLJIT_S390X_RRE(lhr, 0xb9270000) +SLJIT_S390X_RRE(lghr, 0xb9070000) + +/* LOAD LOGICAL */ +SLJIT_S390X_RRE(llgfr, 0xb9160000) + +/* LOAD LOGICAL CHARACTER */ +SLJIT_S390X_RRE(llcr, 0xb9940000) +SLJIT_S390X_RRE(llgcr, 0xb9840000) + +/* LOAD LOGICAL HALFWORD */ +SLJIT_S390X_RRE(llhr, 0xb9950000) +SLJIT_S390X_RRE(llghr, 0xb9850000) + +/* MULTIPLY LOGICAL */ +SLJIT_S390X_RRE(mlgr, 0xb9860000) + +/* MULTIPLY SINGLE */ +SLJIT_S390X_RRE(msr, 0xb2520000) +SLJIT_S390X_RRE(msgr, 0xb90c0000) +SLJIT_S390X_RRE(msgfr, 0xb91c0000) + +/* OR */ +SLJIT_S390X_RRE(ogr, 0xb9810000) + +/* SUBTRACT */ +SLJIT_S390X_RRE(sgr, 0xb9090000) + +/* SUBTRACT LOGICAL */ +SLJIT_S390X_RRE(slgr, 0xb90b0000) + +/* SUBTRACT LOGICAL WITH BORROW */ +SLJIT_S390X_RRE(slbr, 0xb9990000) +SLJIT_S390X_RRE(slbgr, 0xb9890000) + +#undef SLJIT_S390X_RRE + +/* RI-a form instructions */ +#define SLJIT_S390X_RIA(name, pattern, imm_type) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, imm_type imm) \ +{ \ + return (pattern) | ((reg & 0xf) << 20) | (imm & 0xffff); \ +} + +/* ADD HALFWORD IMMEDIATE */ +SLJIT_S390X_RIA(ahi, 0xa70a0000, sljit_s16) +SLJIT_S390X_RIA(aghi, 0xa70b0000, sljit_s16) + +/* COMPARE HALFWORD IMMEDIATE */ +SLJIT_S390X_RIA(chi, 0xa70e0000, sljit_s16) +SLJIT_S390X_RIA(cghi, 0xa70f0000, sljit_s16) + +/* LOAD HALFWORD IMMEDIATE */ +SLJIT_S390X_RIA(lhi, 0xa7080000, sljit_s16) +SLJIT_S390X_RIA(lghi, 0xa7090000, sljit_s16) + +/* LOAD LOGICAL IMMEDIATE */ +SLJIT_S390X_RIA(llihh, 0xa50c0000, sljit_u16) +SLJIT_S390X_RIA(llihl, 0xa50d0000, sljit_u16) +SLJIT_S390X_RIA(llilh, 0xa50e0000, sljit_u16) +SLJIT_S390X_RIA(llill, 0xa50f0000, sljit_u16) + +/* MULTIPLY HALFWORD IMMEDIATE */ +SLJIT_S390X_RIA(mhi, 0xa70c0000, sljit_s16) +SLJIT_S390X_RIA(mghi, 0xa70d0000, sljit_s16) + +/* OR IMMEDIATE */ +SLJIT_S390X_RIA(oilh, 0xa50a0000, sljit_u16) + +/* TEST UNDER MASK */ +SLJIT_S390X_RIA(tmlh, 0xa7000000, sljit_u16) + +#undef SLJIT_S390X_RIA + +/* RIL-a form instructions (requires extended immediate facility) */ +#define SLJIT_S390X_RILA(name, pattern, imm_type) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, imm_type imm) \ +{ \ + SLJIT_ASSERT(have_eimm()); \ + return (pattern) | ((sljit_ins)(reg & 0xf) << 36) | (imm & 0xffffffff); \ +} + +/* ADD IMMEDIATE */ +SLJIT_S390X_RILA(afi, 0xc20900000000, sljit_s32) +SLJIT_S390X_RILA(agfi, 0xc20800000000, sljit_s32) + +/* ADD IMMEDIATE HIGH */ +SLJIT_S390X_RILA(aih, 0xcc0800000000, sljit_s32) /* TODO(mundaym): high-word facility? */ + +/* ADD LOGICAL IMMEDIATE */ +SLJIT_S390X_RILA(alfi, 0xc20b00000000, sljit_u32) +SLJIT_S390X_RILA(algfi, 0xc20a00000000, sljit_u32) + +/* AND IMMEDIATE */ +SLJIT_S390X_RILA(nihf, 0xc00a00000000, sljit_u32) +SLJIT_S390X_RILA(nilf, 0xc00b00000000, sljit_u32) + +/* COMPARE IMMEDIATE */ +SLJIT_S390X_RILA(cfi, 0xc20d00000000, sljit_s32) +SLJIT_S390X_RILA(cgfi, 0xc20c00000000, sljit_s32) + +/* COMPARE IMMEDIATE HIGH */ +SLJIT_S390X_RILA(cih, 0xcc0d00000000, sljit_s32) /* TODO(mundaym): high-word facility? */ + +/* COMPARE LOGICAL IMMEDIATE */ +SLJIT_S390X_RILA(clfi, 0xc20f00000000, sljit_u32) +SLJIT_S390X_RILA(clgfi, 0xc20e00000000, sljit_u32) + +/* EXCLUSIVE OR IMMEDIATE */ +SLJIT_S390X_RILA(xilf, 0xc00700000000, sljit_u32) + +/* INSERT IMMEDIATE */ +SLJIT_S390X_RILA(iihf, 0xc00800000000, sljit_u32) +SLJIT_S390X_RILA(iilf, 0xc00900000000, sljit_u32) + +/* LOAD IMMEDIATE */ +SLJIT_S390X_RILA(lgfi, 0xc00100000000, sljit_s32) + +/* LOAD LOGICAL IMMEDIATE */ +SLJIT_S390X_RILA(llihf, 0xc00e00000000, sljit_u32) +SLJIT_S390X_RILA(llilf, 0xc00f00000000, sljit_u32) + +/* OR IMMEDIATE */ +SLJIT_S390X_RILA(oilf, 0xc00d00000000, sljit_u32) + +#undef SLJIT_S390X_RILA + +/* RX-a form instructions */ +#define SLJIT_S390X_RXA(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr r, sljit_u16 d, sljit_gpr x, sljit_gpr b) \ +{ \ + sljit_ins ri, xi, bi, di; \ +\ + SLJIT_ASSERT((d & 0xfff) == d); \ + ri = (sljit_ins)(r & 0xf) << 20; \ + xi = (sljit_ins)(x & 0xf) << 16; \ + bi = (sljit_ins)(b & 0xf) << 12; \ + di = (sljit_ins)(d & 0xfff); \ +\ + return (pattern) | ri | xi | bi | di; \ +} + +/* ADD */ +SLJIT_S390X_RXA(a, 0x5a000000) + +/* ADD LOGICAL */ +SLJIT_S390X_RXA(al, 0x5e000000) + +/* AND */ +SLJIT_S390X_RXA(n, 0x54000000) + +/* EXCLUSIVE OR */ +SLJIT_S390X_RXA(x, 0x57000000) + +/* LOAD */ +SLJIT_S390X_RXA(l, 0x58000000) + +/* LOAD ADDRESS */ +SLJIT_S390X_RXA(la, 0x41000000) + +/* LOAD HALFWORD */ +SLJIT_S390X_RXA(lh, 0x48000000) + +/* MULTIPLY SINGLE */ +SLJIT_S390X_RXA(ms, 0x71000000) + +/* OR */ +SLJIT_S390X_RXA(o, 0x56000000) + +/* STORE */ +SLJIT_S390X_RXA(st, 0x50000000) + +/* STORE CHARACTER */ +SLJIT_S390X_RXA(stc, 0x42000000) + +/* STORE HALFWORD */ +SLJIT_S390X_RXA(sth, 0x40000000) + +/* SUBTRACT */ +SLJIT_S390X_RXA(s, 0x5b000000) + +/* SUBTRACT LOGICAL */ +SLJIT_S390X_RXA(sl, 0x5f000000) + +#undef SLJIT_S390X_RXA + +/* RXY-a instructions */ +#define SLJIT_S390X_RXYA(name, pattern, cond) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b) \ +{ \ + sljit_ins ri, xi, bi, di; \ +\ + SLJIT_ASSERT(cond); \ + ri = (sljit_ins)(r & 0xf) << 36; \ + xi = (sljit_ins)(x & 0xf) << 32; \ + bi = (sljit_ins)(b & 0xf) << 28; \ + di = (sljit_ins)disp_s20(d) << 8; \ +\ + return (pattern) | ri | xi | bi | di; \ +} + +/* ADD */ +SLJIT_S390X_RXYA(ay, 0xe3000000005a, have_ldisp()) +SLJIT_S390X_RXYA(ag, 0xe30000000008, 1) + +/* ADD LOGICAL */ +SLJIT_S390X_RXYA(aly, 0xe3000000005e, have_ldisp()) +SLJIT_S390X_RXYA(alg, 0xe3000000000a, 1) + +/* ADD LOGICAL WITH CARRY */ +SLJIT_S390X_RXYA(alc, 0xe30000000098, 1) +SLJIT_S390X_RXYA(alcg, 0xe30000000088, 1) + +/* AND */ +SLJIT_S390X_RXYA(ny, 0xe30000000054, have_ldisp()) +SLJIT_S390X_RXYA(ng, 0xe30000000080, 1) + +/* EXCLUSIVE OR */ +SLJIT_S390X_RXYA(xy, 0xe30000000057, have_ldisp()) +SLJIT_S390X_RXYA(xg, 0xe30000000082, 1) + +/* LOAD */ +SLJIT_S390X_RXYA(ly, 0xe30000000058, have_ldisp()) +SLJIT_S390X_RXYA(lg, 0xe30000000004, 1) +SLJIT_S390X_RXYA(lgf, 0xe30000000014, 1) + +/* LOAD BYTE */ +SLJIT_S390X_RXYA(lb, 0xe30000000076, have_ldisp()) +SLJIT_S390X_RXYA(lgb, 0xe30000000077, have_ldisp()) + +/* LOAD HALFWORD */ +SLJIT_S390X_RXYA(lhy, 0xe30000000078, have_ldisp()) +SLJIT_S390X_RXYA(lgh, 0xe30000000015, 1) + +/* LOAD LOGICAL */ +SLJIT_S390X_RXYA(llgf, 0xe30000000016, 1) + +/* LOAD LOGICAL CHARACTER */ +SLJIT_S390X_RXYA(llc, 0xe30000000094, have_eimm()) +SLJIT_S390X_RXYA(llgc, 0xe30000000090, 1) + +/* LOAD LOGICAL HALFWORD */ +SLJIT_S390X_RXYA(llh, 0xe30000000095, have_eimm()) +SLJIT_S390X_RXYA(llgh, 0xe30000000091, 1) + +/* MULTIPLY SINGLE */ +SLJIT_S390X_RXYA(msy, 0xe30000000051, have_ldisp()) +SLJIT_S390X_RXYA(msg, 0xe3000000000c, 1) + +/* OR */ +SLJIT_S390X_RXYA(oy, 0xe30000000056, have_ldisp()) +SLJIT_S390X_RXYA(og, 0xe30000000081, 1) + +/* STORE */ +SLJIT_S390X_RXYA(sty, 0xe30000000050, have_ldisp()) +SLJIT_S390X_RXYA(stg, 0xe30000000024, 1) + +/* STORE CHARACTER */ +SLJIT_S390X_RXYA(stcy, 0xe30000000072, have_ldisp()) + +/* STORE HALFWORD */ +SLJIT_S390X_RXYA(sthy, 0xe30000000070, have_ldisp()) + +/* SUBTRACT */ +SLJIT_S390X_RXYA(sy, 0xe3000000005b, have_ldisp()) +SLJIT_S390X_RXYA(sg, 0xe30000000009, 1) + +/* SUBTRACT LOGICAL */ +SLJIT_S390X_RXYA(sly, 0xe3000000005f, have_ldisp()) +SLJIT_S390X_RXYA(slg, 0xe3000000000b, 1) + +/* SUBTRACT LOGICAL WITH BORROW */ +SLJIT_S390X_RXYA(slb, 0xe30000000099, 1) +SLJIT_S390X_RXYA(slbg, 0xe30000000089, 1) + +#undef SLJIT_S390X_RXYA + +/* RS-a instructions */ +#define SLJIT_S390X_RSA(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, sljit_sw d, sljit_gpr b) \ +{ \ + sljit_ins r1 = (sljit_ins)(reg & 0xf) << 20; \ + sljit_ins b2 = (sljit_ins)(b & 0xf) << 12; \ + sljit_ins d2 = (sljit_ins)(d & 0xfff); \ + return (pattern) | r1 | b2 | d2; \ +} + +/* SHIFT LEFT SINGLE LOGICAL */ +SLJIT_S390X_RSA(sll, 0x89000000) + +/* SHIFT RIGHT SINGLE */ +SLJIT_S390X_RSA(sra, 0x8a000000) + +/* SHIFT RIGHT SINGLE LOGICAL */ +SLJIT_S390X_RSA(srl, 0x88000000) + +#undef SLJIT_S390X_RSA + +/* RSY-a instructions */ +#define SLJIT_S390X_RSYA(name, pattern, cond) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_sw d, sljit_gpr b) \ +{ \ + sljit_ins r1, r3, b2, d2; \ +\ + SLJIT_ASSERT(cond); \ + r1 = (sljit_ins)(dst & 0xf) << 36; \ + r3 = (sljit_ins)(src & 0xf) << 32; \ + b2 = (sljit_ins)(b & 0xf) << 28; \ + d2 = (sljit_ins)disp_s20(d) << 8; \ +\ + return (pattern) | r1 | r3 | b2 | d2; \ +} + +/* LOAD MULTIPLE */ +SLJIT_S390X_RSYA(lmg, 0xeb0000000004, 1) + +/* SHIFT LEFT LOGICAL */ +SLJIT_S390X_RSYA(sllg, 0xeb000000000d, 1) + +/* SHIFT RIGHT SINGLE */ +SLJIT_S390X_RSYA(srag, 0xeb000000000a, 1) + +/* SHIFT RIGHT SINGLE LOGICAL */ +SLJIT_S390X_RSYA(srlg, 0xeb000000000c, 1) + +/* STORE MULTIPLE */ +SLJIT_S390X_RSYA(stmg, 0xeb0000000024, 1) + +#undef SLJIT_S390X_RSYA + +/* RIE-f instructions (require general-instructions-extension facility) */ +#define SLJIT_S390X_RIEF(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_u8 start, sljit_u8 end, sljit_u8 rot) \ +{ \ + sljit_ins r1, r2, i3, i4, i5; \ +\ + SLJIT_ASSERT(have_genext()); \ + r1 = (sljit_ins)(dst & 0xf) << 36; \ + r2 = (sljit_ins)(src & 0xf) << 32; \ + i3 = (sljit_ins)start << 24; \ + i4 = (sljit_ins)end << 16; \ + i5 = (sljit_ins)rot << 8; \ +\ + return (pattern) | r1 | r2 | i3 | i4 | i5; \ +} + +/* ROTATE THEN AND SELECTED BITS */ +/* SLJIT_S390X_RIEF(rnsbg, 0xec0000000054) */ + +/* ROTATE THEN EXCLUSIVE OR SELECTED BITS */ +/* SLJIT_S390X_RIEF(rxsbg, 0xec0000000057) */ + +/* ROTATE THEN OR SELECTED BITS */ +SLJIT_S390X_RIEF(rosbg, 0xec0000000056) + +/* ROTATE THEN INSERT SELECTED BITS */ +/* SLJIT_S390X_RIEF(risbg, 0xec0000000055) */ +/* SLJIT_S390X_RIEF(risbgn, 0xec0000000059) */ + +/* ROTATE THEN INSERT SELECTED BITS HIGH */ +SLJIT_S390X_RIEF(risbhg, 0xec000000005d) + +/* ROTATE THEN INSERT SELECTED BITS LOW */ +/* SLJIT_S390X_RIEF(risblg, 0xec0000000051) */ + +#undef SLJIT_S390X_RIEF + +/* RRF-a instructions */ +#define SLJIT_S390X_RRFA(name, pattern, cond) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src1, sljit_gpr src2) \ +{ \ + sljit_ins r1, r2, r3; \ +\ + SLJIT_ASSERT(cond); \ + r1 = (sljit_ins)(dst & 0xf) << 4; \ + r2 = (sljit_ins)(src1 & 0xf); \ + r3 = (sljit_ins)(src2 & 0xf) << 12; \ +\ + return (pattern) | r3 | r1 | r2; \ +} + +/* MULTIPLY */ +SLJIT_S390X_RRFA(msrkc, 0xb9fd0000, have_misc2()) +SLJIT_S390X_RRFA(msgrkc, 0xb9ed0000, have_misc2()) + +#undef SLJIT_S390X_RRFA + +/* RRF-c instructions (require load/store-on-condition 1 facility) */ +#define SLJIT_S390X_RRFC(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr dst, sljit_gpr src, sljit_uw mask) \ +{ \ + sljit_ins r1, r2, m3; \ +\ + SLJIT_ASSERT(have_lscond1()); \ + r1 = (sljit_ins)(dst & 0xf) << 4; \ + r2 = (sljit_ins)(src & 0xf); \ + m3 = (sljit_ins)(mask & 0xf) << 12; \ +\ + return (pattern) | m3 | r1 | r2; \ +} + +/* LOAD HALFWORD IMMEDIATE ON CONDITION */ +SLJIT_S390X_RRFC(locr, 0xb9f20000) +SLJIT_S390X_RRFC(locgr, 0xb9e20000) + +#undef SLJIT_S390X_RRFC + +/* RIE-g instructions (require load/store-on-condition 2 facility) */ +#define SLJIT_S390X_RIEG(name, pattern) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, sljit_sw imm, sljit_uw mask) \ +{ \ + sljit_ins r1, m3, i2; \ +\ + SLJIT_ASSERT(have_lscond2()); \ + r1 = (sljit_ins)(reg & 0xf) << 36; \ + m3 = (sljit_ins)(mask & 0xf) << 32; \ + i2 = (sljit_ins)(imm & 0xffffL) << 16; \ +\ + return (pattern) | r1 | m3 | i2; \ +} + +/* LOAD HALFWORD IMMEDIATE ON CONDITION */ +SLJIT_S390X_RIEG(lochi, 0xec0000000042) +SLJIT_S390X_RIEG(locghi, 0xec0000000046) + +#undef SLJIT_S390X_RIEG + +#define SLJIT_S390X_RILB(name, pattern, cond) \ +SLJIT_S390X_INSTRUCTION(name, sljit_gpr reg, sljit_sw ri) \ +{ \ + sljit_ins r1, ri2; \ +\ + SLJIT_ASSERT(cond); \ + r1 = (sljit_ins)(reg & 0xf) << 36; \ + ri2 = (sljit_ins)(ri & 0xffffffff); \ +\ + return (pattern) | r1 | ri2; \ +} + +/* BRANCH RELATIVE AND SAVE LONG */ +SLJIT_S390X_RILB(brasl, 0xc00500000000, 1) + +/* LOAD ADDRESS RELATIVE LONG */ +SLJIT_S390X_RILB(larl, 0xc00000000000, 1) + +/* LOAD RELATIVE LONG */ +SLJIT_S390X_RILB(lgrl, 0xc40800000000, have_genext()) + +#undef SLJIT_S390X_RILB + +SLJIT_S390X_INSTRUCTION(br, sljit_gpr target) +{ + return 0x07f0 | target; +} + +SLJIT_S390X_INSTRUCTION(brcl, sljit_uw mask, sljit_sw target) +{ + sljit_ins m1 = (sljit_ins)(mask & 0xf) << 36; + sljit_ins ri2 = (sljit_ins)target & 0xffffffff; + return 0xc00400000000L | m1 | ri2; +} + +SLJIT_S390X_INSTRUCTION(flogr, sljit_gpr dst, sljit_gpr src) +{ + sljit_ins r1 = ((sljit_ins)dst & 0xf) << 8; + sljit_ins r2 = ((sljit_ins)src & 0xf); + SLJIT_ASSERT(have_eimm()); + return 0xb9830000 | r1 | r2; +} + +/* INSERT PROGRAM MASK */ +SLJIT_S390X_INSTRUCTION(ipm, sljit_gpr dst) +{ + return 0xb2220000 | ((sljit_ins)(dst & 0xf) << 4); +} + +/* ROTATE THEN INSERT SELECTED BITS HIGH (ZERO) */ +SLJIT_S390X_INSTRUCTION(risbhgz, sljit_gpr dst, sljit_gpr src, sljit_u8 start, sljit_u8 end, sljit_u8 rot) +{ + return risbhg(dst, src, start, 0x8 | end, rot); +} + +#undef SLJIT_S390X_INSTRUCTION + +/* load condition code as needed to match type */ +static sljit_s32 push_load_cc(struct sljit_compiler *compiler, sljit_s32 type) +{ + type &= ~SLJIT_I32_OP; + switch (type) { + case SLJIT_ZERO: + case SLJIT_NOT_ZERO: + return push_inst(compiler, cih(flag_r, 0)); + break; + default: + return push_inst(compiler, tmlh(flag_r, 0x3000)); + break; + } + return SLJIT_SUCCESS; +} + +static sljit_s32 push_store_zero_flag(struct sljit_compiler *compiler, sljit_s32 op, sljit_gpr source) +{ + /* insert low 32-bits into high 32-bits of flag register */ + FAIL_IF(push_inst(compiler, risbhgz(flag_r, source, 0, 31, 32))); + if (!(op & SLJIT_I32_OP)) { + /* OR high 32-bits with high 32-bits of flag register */ + return push_inst(compiler, rosbg(flag_r, source, 0, 31, 0)); + } + return SLJIT_SUCCESS; +} + +/* load 64-bit immediate into register without clobbering flags */ +static sljit_s32 push_load_imm_inst(struct sljit_compiler *compiler, sljit_gpr target, sljit_sw v) +{ + /* 4 byte instructions */ + if (is_s16(v)) + return push_inst(compiler, lghi(target, (sljit_s16)v)); + + if ((sljit_uw)v == (v & 0x000000000000ffffU)) + return push_inst(compiler, llill(target, (sljit_u16)v)); + + if ((sljit_uw)v == (v & 0x00000000ffff0000U)) + return push_inst(compiler, llilh(target, (sljit_u16)(v >> 16))); + + if ((sljit_uw)v == (v & 0x0000ffff00000000U)) + return push_inst(compiler, llihl(target, (sljit_u16)(v >> 32))); + + if ((sljit_uw)v == (v & 0xffff000000000000U)) + return push_inst(compiler, llihh(target, (sljit_u16)(v >> 48))); + + /* 6 byte instructions (requires extended immediate facility) */ + if (have_eimm()) { + if (is_s32(v)) + return push_inst(compiler, lgfi(target, (sljit_s32)v)); + + if ((sljit_uw)v == (v & 0x00000000ffffffffU)) + return push_inst(compiler, llilf(target, (sljit_u32)v)); + + if ((sljit_uw)v == (v & 0xffffffff00000000U)) + return push_inst(compiler, llihf(target, (sljit_u32)(v >> 32))); + + FAIL_IF(push_inst(compiler, llilf(target, (sljit_u32)v))); + return push_inst(compiler, iihf(target, (sljit_u32)(v >> 32))); + } + /* TODO(mundaym): instruction sequences that don't use extended immediates */ + abort(); +} + +struct addr { + sljit_gpr base; + sljit_gpr index; + sljit_sw offset; +}; + +/* transform memory operand into D(X,B) form with a signed 20-bit offset */ +static sljit_s32 make_addr_bxy(struct sljit_compiler *compiler, + struct addr *addr, sljit_s32 mem, sljit_sw off, + sljit_gpr tmp /* clobbered, must not be r0 */) +{ + sljit_gpr base = r0; + sljit_gpr index = r0; + + SLJIT_ASSERT(tmp != r0); + if (mem & REG_MASK) + base = gpr(mem & REG_MASK); + + if (mem & OFFS_REG_MASK) { + index = gpr(OFFS_REG(mem)); + if (off != 0) { + /* shift and put the result into tmp */ + SLJIT_ASSERT(0 <= off && off < 64); + FAIL_IF(push_inst(compiler, sllg(tmp, index, off, 0))); + index = tmp; + off = 0; /* clear offset */ + } + } + else if (!is_s20(off)) { + FAIL_IF(push_load_imm_inst(compiler, tmp, off)); + index = tmp; + off = 0; /* clear offset */ + } + addr->base = base; + addr->index = index; + addr->offset = off; + return SLJIT_SUCCESS; +} + +/* transform memory operand into D(X,B) form with an unsigned 12-bit offset */ +static sljit_s32 make_addr_bx(struct sljit_compiler *compiler, + struct addr *addr, sljit_s32 mem, sljit_sw off, + sljit_gpr tmp /* clobbered, must not be r0 */) +{ + sljit_gpr base = r0; + sljit_gpr index = r0; + + SLJIT_ASSERT(tmp != r0); + if (mem & REG_MASK) + base = gpr(mem & REG_MASK); + + if (mem & OFFS_REG_MASK) { + index = gpr(OFFS_REG(mem)); + if (off != 0) { + /* shift and put the result into tmp */ + SLJIT_ASSERT(0 <= off && off < 64); + FAIL_IF(push_inst(compiler, sllg(tmp, index, off, 0))); + index = tmp; + off = 0; /* clear offset */ + } + } + else if (!is_u12(off)) { + FAIL_IF(push_load_imm_inst(compiler, tmp, off)); + index = tmp; + off = 0; /* clear offset */ + } + addr->base = base; + addr->index = index; + addr->offset = off; + return SLJIT_SUCCESS; +} + +#define EVAL(op, r, addr) op(r, addr.offset, addr.index, addr.base) +#define WHEN(cond, r, i1, i2, addr) \ + (cond) ? EVAL(i1, r, addr) : EVAL(i2, r, addr) + +static sljit_s32 load_word(struct sljit_compiler *compiler, sljit_gpr dst, + sljit_s32 src, sljit_sw srcw, + sljit_gpr tmp /* clobbered */, sljit_s32 is_32bit) +{ + struct addr addr; + sljit_ins ins; + + SLJIT_ASSERT(src & SLJIT_MEM); + if (have_ldisp() || !is_32bit) + FAIL_IF(make_addr_bxy(compiler, &addr, src, srcw, tmp)); + else + FAIL_IF(make_addr_bx(compiler, &addr, src, srcw, tmp)); + + if (is_32bit) + ins = WHEN(is_u12(addr.offset), dst, l, ly, addr); + else + ins = lg(dst, addr.offset, addr.index, addr.base); + + return push_inst(compiler, ins); +} + +static sljit_s32 store_word(struct sljit_compiler *compiler, sljit_gpr src, + sljit_s32 dst, sljit_sw dstw, + sljit_gpr tmp /* clobbered */, sljit_s32 is_32bit) +{ + struct addr addr; + sljit_ins ins; + + SLJIT_ASSERT(dst & SLJIT_MEM); + if (have_ldisp() || !is_32bit) + FAIL_IF(make_addr_bxy(compiler, &addr, dst, dstw, tmp)); + else + FAIL_IF(make_addr_bx(compiler, &addr, dst, dstw, tmp)); + + if (is_32bit) + ins = WHEN(is_u12(addr.offset), src, st, sty, addr); + else + ins = stg(src, addr.offset, addr.index, addr.base); + + return push_inst(compiler, ins); +} + +#undef WHEN + +SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler) +{ + struct sljit_label *label; + struct sljit_jump *jump; + struct sljit_s390x_const *const_; + struct sljit_put_label *put_label; + sljit_sw executable_offset; + sljit_uw ins_size = 0; /* instructions */ + sljit_uw pool_size = 0; /* literal pool */ + sljit_uw pad_size; + sljit_uw i, j = 0; + struct sljit_memory_fragment *buf; + void *code, *code_ptr; + sljit_uw *pool, *pool_ptr; + + sljit_uw source; + sljit_sw offset; /* TODO(carenas): only need 32 bit */ + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_generate_code(compiler)); + reverse_buf(compiler); + + /* branch handling */ + label = compiler->labels; + jump = compiler->jumps; + put_label = compiler->put_labels; + + /* TODO(carenas): compiler->executable_size could be calculated + * before to avoid the following loop (except for + * pool_size) + */ + /* calculate the size of the code */ + for (buf = compiler->buf; buf != NULL; buf = buf->next) { + sljit_uw len = buf->used_size / sizeof(sljit_ins); + sljit_ins *ibuf = (sljit_ins *)buf->memory; + for (i = 0; i < len; ++i, ++j) { + sljit_ins ins = ibuf[i]; + + /* TODO(carenas): instruction tag vs size/addr == j + * using instruction tags for const is creative + * but unlike all other architectures, and is not + * done consistently for all other objects. + * This might need reviewing later. + */ + if (ins & sljit_ins_const) { + pool_size += sizeof(*pool); + ins &= ~sljit_ins_const; + } + if (label && label->size == j) { + label->size = ins_size; + label = label->next; + } + if (jump && jump->addr == j) { + if ((jump->flags & SLJIT_REWRITABLE_JUMP) || (jump->flags & JUMP_ADDR)) { + /* encoded: */ + /* brasl %r14, <rel_addr> (or brcl <mask>, <rel_addr>) */ + /* replace with: */ + /* lgrl %r1, <pool_addr> */ + /* bras %r14, %r1 (or bcr <mask>, %r1) */ + pool_size += sizeof(*pool); + ins_size += 2; + } + jump = jump->next; + } + if (put_label && put_label->addr == j) { + pool_size += sizeof(*pool); + put_label = put_label->next; + } + ins_size += sizeof_ins(ins); + } + } + + /* emit trailing label */ + if (label && label->size == j) { + label->size = ins_size; + label = label->next; + } + + SLJIT_ASSERT(!label); + SLJIT_ASSERT(!jump); + SLJIT_ASSERT(!put_label); + + /* pad code size to 8 bytes so is accessible with half word offsets */ + /* the literal pool needs to be doubleword aligned */ + pad_size = ((ins_size + 7UL) & ~7UL) - ins_size; + SLJIT_ASSERT(pad_size < 8UL); + + /* allocate target buffer */ + code = SLJIT_MALLOC_EXEC(ins_size + pad_size + pool_size, + compiler->exec_allocator_data); + PTR_FAIL_WITH_EXEC_IF(code); + code_ptr = code; + executable_offset = SLJIT_EXEC_OFFSET(code); + + /* TODO(carenas): pool is optional, and the ABI recommends it to + * be created before the function code, instead of + * globally; if generated code is too big could + * need offsets bigger than 32bit words and asser() + */ + pool = (sljit_uw *)((sljit_uw)code + ins_size + pad_size); + pool_ptr = pool; + const_ = (struct sljit_s390x_const *)compiler->consts; + + /* update label addresses */ + label = compiler->labels; + while (label) { + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET( + (sljit_uw)code_ptr + label->size, executable_offset); + label = label->next; + } + + /* reset jumps */ + jump = compiler->jumps; + put_label = compiler->put_labels; + + /* emit the code */ + j = 0; + for (buf = compiler->buf; buf != NULL; buf = buf->next) { + sljit_uw len = buf->used_size / sizeof(sljit_ins); + sljit_ins *ibuf = (sljit_ins *)buf->memory; + for (i = 0; i < len; ++i, ++j) { + sljit_ins ins = ibuf[i]; + if (ins & sljit_ins_const) { + /* clear the const tag */ + ins &= ~sljit_ins_const; + + /* update instruction with relative address of constant */ + source = (sljit_uw)code_ptr; + offset = (sljit_uw)pool_ptr - source; + SLJIT_ASSERT(!(offset & 1)); + offset >>= 1; /* halfword (not byte) offset */ + SLJIT_ASSERT(is_s32(offset)); + ins |= (sljit_ins)offset & 0xffffffff; + + /* update address */ + const_->const_.addr = (sljit_uw)pool_ptr; + + /* store initial value into pool and update pool address */ + *(pool_ptr++) = const_->init_value; + + /* move to next constant */ + const_ = (struct sljit_s390x_const *)const_->const_.next; + } + if (jump && jump->addr == j) { + sljit_sw target = (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target; + if ((jump->flags & SLJIT_REWRITABLE_JUMP) || (jump->flags & JUMP_ADDR)) { + jump->addr = (sljit_uw)pool_ptr; + + /* load address into tmp1 */ + source = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + offset = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(pool_ptr, executable_offset) - source; + SLJIT_ASSERT(!(offset & 1)); + offset >>= 1; + SLJIT_ASSERT(is_s32(offset)); + encode_inst(&code_ptr, + lgrl(tmp1, offset & 0xffffffff)); + + /* store jump target into pool and update pool address */ + *(pool_ptr++) = target; + + /* branch to tmp1 */ + sljit_ins op = (ins >> 32) & 0xf; + sljit_ins arg = (ins >> 36) & 0xf; + switch (op) { + case 4: /* brcl -> bcr */ + ins = bcr(arg, tmp1); + break; + case 5: /* brasl -> basr */ + ins = basr(arg, tmp1); + break; + default: + abort(); + } + } + else { + jump->addr = (sljit_uw)code_ptr + 2; + source = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + offset = target - source; + + /* offset must be halfword aligned */ + SLJIT_ASSERT(!(offset & 1)); + offset >>= 1; + SLJIT_ASSERT(is_s32(offset)); /* TODO(mundaym): handle arbitrary offsets */ + + /* patch jump target */ + ins |= (sljit_ins)offset & 0xffffffff; + } + jump = jump->next; + } + if (put_label && put_label->addr == j) { + source = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; + + /* store target into pool */ + *pool_ptr = put_label->label->addr; + offset = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(pool_ptr, executable_offset) - source; + pool_ptr++; + + SLJIT_ASSERT(!(offset & 1)); + offset >>= 1; + SLJIT_ASSERT(is_s32(offset)); + ins |= (sljit_ins)offset & 0xffffffff; + + put_label = put_label->next; + } + encode_inst(&code_ptr, ins); + } + } + SLJIT_ASSERT((sljit_u8 *)code + ins_size == code_ptr); + SLJIT_ASSERT((sljit_u8 *)pool + pool_size == (sljit_u8 *)pool_ptr); + + compiler->error = SLJIT_ERR_COMPILED; + compiler->executable_offset = executable_offset; + compiler->executable_size = ins_size; + code = SLJIT_ADD_EXEC_OFFSET(code, executable_offset); + code_ptr = SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); + return code; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) +{ + /* TODO(mundaym): implement all */ + switch (feature_type) { + case SLJIT_HAS_CLZ: + return have_eimm() ? 1 : 0; /* FLOGR instruction */ + case SLJIT_HAS_CMOV: + return have_lscond1() ? 1 : 0; + case SLJIT_HAS_FPU: + return 0; + } + return 0; +} + +/* --------------------------------------------------------------------- */ +/* Entry, exit */ +/* --------------------------------------------------------------------- */ + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler, + sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, + sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) +{ + sljit_s32 args = get_arg_count(arg_types); + sljit_sw frame_size; + + CHECK_ERROR(); + CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size)); + set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size); + + /* saved registers go in callee allocated save area */ + compiler->local_size = (local_size + 0xf) & ~0xf; + frame_size = compiler->local_size + SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE; + + FAIL_IF(push_inst(compiler, stmg(r6, r15, r6 * sizeof(sljit_sw), r15))); /* save registers TODO(MGM): optimize */ + if (frame_size != 0) { + if (is_s16(-frame_size)) + FAIL_IF(push_inst(compiler, aghi(r15, -((sljit_s16)frame_size)))); + else if (is_s32(-frame_size)) + FAIL_IF(push_inst(compiler, agfi(r15, -((sljit_s32)frame_size)))); + else { + FAIL_IF(push_load_imm_inst(compiler, tmp1, -frame_size)); + FAIL_IF(push_inst(compiler, la(r15, 0, tmp1, r15))); + } + } + + if (args >= 1) + FAIL_IF(push_inst(compiler, lgr(gpr(SLJIT_S0), gpr(SLJIT_R0)))); + if (args >= 2) + FAIL_IF(push_inst(compiler, lgr(gpr(SLJIT_S1), gpr(SLJIT_R1)))); + if (args >= 3) + FAIL_IF(push_inst(compiler, lgr(gpr(SLJIT_S2), gpr(SLJIT_R2)))); + SLJIT_ASSERT(args < 4); + + return SLJIT_SUCCESS; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler, + sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, + sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) +{ + CHECK_ERROR(); + CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size)); + set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size); + + /* TODO(mundaym): stack space for saved floating point registers */ + compiler->local_size = (local_size + 0xf) & ~0xf; + return SLJIT_SUCCESS; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw) +{ + sljit_sw size; + sljit_gpr end; + + CHECK_ERROR(); + CHECK(check_sljit_emit_return(compiler, op, src, srcw)); + + FAIL_IF(emit_mov_before_return(compiler, op, src, srcw)); + + size = compiler->local_size + SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE + (r6 * sizeof(sljit_sw)); + if (!is_s20(size)) { + FAIL_IF(push_load_imm_inst(compiler, tmp1, compiler->local_size + SLJIT_S390X_DEFAULT_STACK_FRAME_SIZE)); + FAIL_IF(push_inst(compiler, la(r15, 0, tmp1, r15))); + size = r6 * sizeof(sljit_sw); + end = r14; /* r15 has been restored already */ + } + else + end = r15; + + FAIL_IF(push_inst(compiler, lmg(r6, end, size, r15))); /* restore registers TODO(MGM): optimize */ + FAIL_IF(push_inst(compiler, br(r14))); /* return */ + + return SLJIT_SUCCESS; +} + +/* --------------------------------------------------------------------- */ +/* Operators */ +/* --------------------------------------------------------------------- */ + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op) +{ + sljit_gpr arg0 = gpr(SLJIT_R0); + sljit_gpr arg1 = gpr(SLJIT_R1); + + CHECK_ERROR(); + CHECK(check_sljit_emit_op0(compiler, op)); + + op = GET_OPCODE(op) | (op & SLJIT_I32_OP); + switch (op) { + case SLJIT_BREAKPOINT: + /* TODO(mundaym): insert real breakpoint? */ + case SLJIT_NOP: + return push_inst(compiler, 0x0700 /* 2-byte nop */); + case SLJIT_LMUL_UW: + FAIL_IF(push_inst(compiler, mlgr(arg0, arg0))); + break; + case SLJIT_LMUL_SW: + /* signed multiplication from: */ + /* Hacker's Delight, Second Edition: Chapter 8-3. */ + FAIL_IF(push_inst(compiler, srag(tmp0, arg0, 63, 0))); + FAIL_IF(push_inst(compiler, srag(tmp1, arg1, 63, 0))); + FAIL_IF(push_inst(compiler, ngr(tmp0, arg1))); + FAIL_IF(push_inst(compiler, ngr(tmp1, arg0))); + + /* unsigned multiplication */ + FAIL_IF(push_inst(compiler, mlgr(arg0, arg0))); + + FAIL_IF(push_inst(compiler, sgr(arg0, tmp0))); + FAIL_IF(push_inst(compiler, sgr(arg0, tmp1))); + break; + case SLJIT_DIV_U32: + case SLJIT_DIVMOD_U32: + FAIL_IF(push_inst(compiler, lhi(tmp0, 0))); + FAIL_IF(push_inst(compiler, lr(tmp1, arg0))); + FAIL_IF(push_inst(compiler, dlr(tmp0, arg1))); + FAIL_IF(push_inst(compiler, lr(arg0, tmp1))); /* quotient */ + if (op == SLJIT_DIVMOD_U32) + return push_inst(compiler, lr(arg1, tmp0)); /* remainder */ + + return SLJIT_SUCCESS; + case SLJIT_DIV_S32: + case SLJIT_DIVMOD_S32: + FAIL_IF(push_inst(compiler, lhi(tmp0, 0))); + FAIL_IF(push_inst(compiler, lr(tmp1, arg0))); + FAIL_IF(push_inst(compiler, dr(tmp0, arg1))); + FAIL_IF(push_inst(compiler, lr(arg0, tmp1))); /* quotient */ + if (op == SLJIT_DIVMOD_S32) + return push_inst(compiler, lr(arg1, tmp0)); /* remainder */ + + return SLJIT_SUCCESS; + case SLJIT_DIV_UW: + case SLJIT_DIVMOD_UW: + FAIL_IF(push_inst(compiler, lghi(tmp0, 0))); + FAIL_IF(push_inst(compiler, lgr(tmp1, arg0))); + FAIL_IF(push_inst(compiler, dlgr(tmp0, arg1))); + FAIL_IF(push_inst(compiler, lgr(arg0, tmp1))); /* quotient */ + if (op == SLJIT_DIVMOD_UW) + return push_inst(compiler, lgr(arg1, tmp0)); /* remainder */ + + return SLJIT_SUCCESS; + case SLJIT_DIV_SW: + case SLJIT_DIVMOD_SW: + FAIL_IF(push_inst(compiler, lgr(tmp1, arg0))); + FAIL_IF(push_inst(compiler, dsgr(tmp0, arg1))); + FAIL_IF(push_inst(compiler, lgr(arg0, tmp1))); /* quotient */ + if (op == SLJIT_DIVMOD_SW) + return push_inst(compiler, lgr(arg1, tmp0)); /* remainder */ + + return SLJIT_SUCCESS; + case SLJIT_ENDBR: + return SLJIT_SUCCESS; + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; + default: + SLJIT_UNREACHABLE(); + } + /* swap result registers */ + FAIL_IF(push_inst(compiler, lgr(tmp0, arg0))); + FAIL_IF(push_inst(compiler, lgr(arg0, arg1))); + return push_inst(compiler, lgr(arg1, tmp0)); +} + +/* LEVAL will be defined later with different parameters as needed */ +#define WHEN2(cond, i1, i2) (cond) ? LEVAL(i1) : LEVAL(i2) + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 dst, sljit_sw dstw, + sljit_s32 src, sljit_sw srcw) +{ + sljit_ins ins; + struct addr mem; + sljit_gpr dst_r; + sljit_gpr src_r; + sljit_s32 opcode = GET_OPCODE(op); + + CHECK_ERROR(); + CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + ADJUST_LOCAL_OFFSET(src, srcw); + + if ((dst == SLJIT_UNUSED) && !HAS_FLAGS(op)) { + /* TODO(carenas): implement prefetch? */ + return SLJIT_SUCCESS; + } + if (opcode >= SLJIT_MOV && opcode <= SLJIT_MOV_P) { + /* LOAD REGISTER */ + if (FAST_IS_REG(dst) && FAST_IS_REG(src)) { + dst_r = gpr(dst); + src_r = gpr(src); + switch (opcode | (op & SLJIT_I32_OP)) { + /* 32-bit */ + case SLJIT_MOV32_U8: + ins = llcr(dst_r, src_r); + break; + case SLJIT_MOV32_S8: + ins = lbr(dst_r, src_r); + break; + case SLJIT_MOV32_U16: + ins = llhr(dst_r, src_r); + break; + case SLJIT_MOV32_S16: + ins = lhr(dst_r, src_r); + break; + case SLJIT_MOV32: + ins = lr(dst_r, src_r); + break; + /* 64-bit */ + case SLJIT_MOV_U8: + ins = llgcr(dst_r, src_r); + break; + case SLJIT_MOV_S8: + ins = lgbr(dst_r, src_r); + break; + case SLJIT_MOV_U16: + ins = llghr(dst_r, src_r); + break; + case SLJIT_MOV_S16: + ins = lghr(dst_r, src_r); + break; + case SLJIT_MOV_U32: + ins = llgfr(dst_r, src_r); + break; + case SLJIT_MOV_S32: + ins = lgfr(dst_r, src_r); + break; + case SLJIT_MOV: + case SLJIT_MOV_P: + ins = lgr(dst_r, src_r); + break; + default: + ins = 0; + SLJIT_UNREACHABLE(); + } + FAIL_IF(push_inst(compiler, ins)); + if (HAS_FLAGS(op)) { + /* only handle zero flag */ + SLJIT_ASSERT(!(op & VARIABLE_FLAG_MASK)); + return push_store_zero_flag(compiler, op, dst_r); + } + return SLJIT_SUCCESS; + } + /* LOAD IMMEDIATE */ + if (FAST_IS_REG(dst) && (src & SLJIT_IMM)) { + switch (opcode) { + case SLJIT_MOV_U8: + srcw = (sljit_sw)((sljit_u8)(srcw)); + break; + case SLJIT_MOV_S8: + srcw = (sljit_sw)((sljit_s8)(srcw)); + break; + case SLJIT_MOV_U16: + srcw = (sljit_sw)((sljit_u16)(srcw)); + break; + case SLJIT_MOV_S16: + srcw = (sljit_sw)((sljit_s16)(srcw)); + break; + case SLJIT_MOV_U32: + srcw = (sljit_sw)((sljit_u32)(srcw)); + break; + case SLJIT_MOV_S32: + srcw = (sljit_sw)((sljit_s32)(srcw)); + break; + } + return push_load_imm_inst(compiler, gpr(dst), srcw); + } + /* LOAD */ + /* TODO(carenas): avoid reg being defined later */ + #define LEVAL(i) EVAL(i, reg, mem) + if (FAST_IS_REG(dst) && (src & SLJIT_MEM)) { + sljit_gpr reg = gpr(dst); + + FAIL_IF(make_addr_bxy(compiler, &mem, src, srcw, tmp1)); + /* TODO(carenas): convert all calls below to LEVAL */ + switch (opcode | (op & SLJIT_I32_OP)) { + case SLJIT_MOV32_U8: + ins = llc(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV32_S8: + ins = lb(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV32_U16: + ins = llh(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV32_S16: + ins = WHEN2(is_u12(mem.offset), lh, lhy); + break; + case SLJIT_MOV32: + ins = WHEN2(is_u12(mem.offset), l, ly); + break; + case SLJIT_MOV_U8: + ins = LEVAL(llgc); + break; + case SLJIT_MOV_S8: + ins = lgb(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV_U16: + ins = LEVAL(llgh); + break; + case SLJIT_MOV_S16: + ins = lgh(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV_U32: + ins = LEVAL(llgf); + break; + case SLJIT_MOV_S32: + ins = lgf(reg, mem.offset, mem.index, mem.base); + break; + case SLJIT_MOV_P: + case SLJIT_MOV: + ins = lg(reg, mem.offset, mem.index, mem.base); + break; + default: + SLJIT_UNREACHABLE(); + } + FAIL_IF(push_inst(compiler, ins)); + if (HAS_FLAGS(op)) { + /* only handle zero flag */ + SLJIT_ASSERT(!(op & VARIABLE_FLAG_MASK)); + return push_store_zero_flag(compiler, op, reg); + } + return SLJIT_SUCCESS; + } + /* STORE and STORE IMMEDIATE */ + if ((dst & SLJIT_MEM) + && (FAST_IS_REG(src) || (src & SLJIT_IMM))) { + sljit_gpr reg = FAST_IS_REG(src) ? gpr(src) : tmp0; + if (src & SLJIT_IMM) { + /* TODO(mundaym): MOVE IMMEDIATE? */ + FAIL_IF(push_load_imm_inst(compiler, reg, srcw)); + } + struct addr mem; + FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1)); + switch (opcode) { + case SLJIT_MOV_U8: + case SLJIT_MOV_S8: + return push_inst(compiler, + WHEN2(is_u12(mem.offset), stc, stcy)); + case SLJIT_MOV_U16: + case SLJIT_MOV_S16: + return push_inst(compiler, + WHEN2(is_u12(mem.offset), sth, sthy)); + case SLJIT_MOV_U32: + case SLJIT_MOV_S32: + return push_inst(compiler, + WHEN2(is_u12(mem.offset), st, sty)); + case SLJIT_MOV_P: + case SLJIT_MOV: + FAIL_IF(push_inst(compiler, LEVAL(stg))); + if (HAS_FLAGS(op)) { + /* only handle zero flag */ + SLJIT_ASSERT(!(op & VARIABLE_FLAG_MASK)); + return push_store_zero_flag(compiler, op, reg); + } + return SLJIT_SUCCESS; + default: + SLJIT_UNREACHABLE(); + } + } + #undef LEVAL + /* MOVE CHARACTERS */ + if ((dst & SLJIT_MEM) && (src & SLJIT_MEM)) { + struct addr mem; + FAIL_IF(make_addr_bxy(compiler, &mem, src, srcw, tmp1)); + switch (opcode) { + case SLJIT_MOV_U8: + case SLJIT_MOV_S8: + FAIL_IF(push_inst(compiler, + EVAL(llgc, tmp0, mem))); + FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1)); + return push_inst(compiler, + EVAL(stcy, tmp0, mem)); + case SLJIT_MOV_U16: + case SLJIT_MOV_S16: + FAIL_IF(push_inst(compiler, + EVAL(llgh, tmp0, mem))); + FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1)); + return push_inst(compiler, + EVAL(sthy, tmp0, mem)); + case SLJIT_MOV_U32: + case SLJIT_MOV_S32: + FAIL_IF(push_inst(compiler, + EVAL(ly, tmp0, mem))); + FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1)); + return push_inst(compiler, + EVAL(sty, tmp0, mem)); + case SLJIT_MOV_P: + case SLJIT_MOV: + FAIL_IF(push_inst(compiler, + EVAL(lg, tmp0, mem))); + FAIL_IF(make_addr_bxy(compiler, &mem, dst, dstw, tmp1)); + FAIL_IF(push_inst(compiler, + EVAL(stg, tmp0, mem))); + if (HAS_FLAGS(op)) { + /* only handle zero flag */ + SLJIT_ASSERT(!(op & VARIABLE_FLAG_MASK)); + return push_store_zero_flag(compiler, op, tmp0); + } + return SLJIT_SUCCESS; + default: + SLJIT_UNREACHABLE(); + } + } + SLJIT_UNREACHABLE(); + } + + SLJIT_ASSERT((src & SLJIT_IMM) == 0); /* no immediates */ + + dst_r = SLOW_IS_REG(dst) ? gpr(REG_MASK & dst) : tmp0; + src_r = FAST_IS_REG(src) ? gpr(REG_MASK & src) : tmp0; + if (src & SLJIT_MEM) + FAIL_IF(load_word(compiler, src_r, src, srcw, tmp1, src & SLJIT_I32_OP)); + + /* TODO(mundaym): optimize loads and stores */ + switch (opcode | (op & SLJIT_I32_OP)) { + case SLJIT_NOT: + /* emulate ~x with x^-1 */ + FAIL_IF(push_load_imm_inst(compiler, tmp1, -1)); + if (src_r != dst_r) + FAIL_IF(push_inst(compiler, lgr(dst_r, src_r))); + + FAIL_IF(push_inst(compiler, xgr(dst_r, tmp1))); + break; + case SLJIT_NOT32: + /* emulate ~x with x^-1 */ + if (have_eimm()) + FAIL_IF(push_inst(compiler, xilf(dst_r, -1))); + else { + FAIL_IF(push_load_imm_inst(compiler, tmp1, -1)); + if (src_r != dst_r) + FAIL_IF(push_inst(compiler, lr(dst_r, src_r))); + + FAIL_IF(push_inst(compiler, xr(dst_r, tmp1))); + } + break; + case SLJIT_NEG: + FAIL_IF(push_inst(compiler, lcgr(dst_r, src_r))); + break; + case SLJIT_NEG32: + FAIL_IF(push_inst(compiler, lcr(dst_r, src_r))); + break; + case SLJIT_CLZ: + if (have_eimm()) { + FAIL_IF(push_inst(compiler, flogr(tmp0, src_r))); /* clobbers tmp1 */ + if (dst_r != tmp0) + FAIL_IF(push_inst(compiler, lgr(dst_r, tmp0))); + } else { + abort(); /* TODO(mundaym): no eimm (?) */ + } + break; + case SLJIT_CLZ32: + if (have_eimm()) { + FAIL_IF(push_inst(compiler, sllg(tmp1, src_r, 32, 0))); + FAIL_IF(push_inst(compiler, iilf(tmp1, 0xffffffff))); + FAIL_IF(push_inst(compiler, flogr(tmp0, tmp1))); /* clobbers tmp1 */ + if (dst_r != tmp0) + FAIL_IF(push_inst(compiler, lr(dst_r, tmp0))); + } else { + abort(); /* TODO(mundaym): no eimm (?) */ + } + break; + default: + SLJIT_UNREACHABLE(); + } + + /* write condition code to emulated flag register */ + if (op & VARIABLE_FLAG_MASK) + FAIL_IF(push_inst(compiler, ipm(flag_r))); + + /* write zero flag to emulated flag register */ + if (op & SLJIT_SET_Z) + FAIL_IF(push_store_zero_flag(compiler, op, dst_r)); + + /* TODO(carenas): doesn't need FAIL_IF */ + if ((dst != SLJIT_UNUSED) && (dst & SLJIT_MEM)) + FAIL_IF(store_word(compiler, dst_r, dst, dstw, tmp1, op & SLJIT_I32_OP)); + + return SLJIT_SUCCESS; +} + +static SLJIT_INLINE int is_commutative(sljit_s32 op) +{ + switch (GET_OPCODE(op)) { + case SLJIT_ADD: + case SLJIT_ADDC: + case SLJIT_MUL: + case SLJIT_AND: + case SLJIT_OR: + case SLJIT_XOR: + return 1; + } + return 0; +} + +static SLJIT_INLINE int is_shift(sljit_s32 op) { + sljit_s32 v = GET_OPCODE(op); + return (v == SLJIT_SHL || v == SLJIT_ASHR || v == SLJIT_LSHR) ? 1 : 0; +} + +static SLJIT_INLINE int sets_signed_flag(sljit_s32 op) +{ + switch (GET_FLAG_TYPE(op)) { + case SLJIT_OVERFLOW: + case SLJIT_NOT_OVERFLOW: + case SLJIT_SIG_LESS: + case SLJIT_SIG_LESS_EQUAL: + case SLJIT_SIG_GREATER: + case SLJIT_SIG_GREATER_EQUAL: + return 1; + } + return 0; +} + +/* Report whether we have an instruction for: + op dst src imm + where dst and src are separate registers. */ +static int have_op_3_imm(sljit_s32 op, sljit_sw imm) { + return 0; /* TODO(mundaym): implement */ +} + +/* Report whether we have an instruction for: + op reg imm + where reg is both a source and the destination. */ +static int have_op_2_imm(sljit_s32 op, sljit_sw imm) { + switch (GET_OPCODE(op) | (op & SLJIT_I32_OP)) { + case SLJIT_ADD32: + case SLJIT_ADD: + if (!HAS_FLAGS(op) || sets_signed_flag(op)) + return have_eimm() ? is_s32(imm) : is_s16(imm); + + return have_eimm() && is_u32(imm); + case SLJIT_MUL32: + case SLJIT_MUL: + /* TODO(mundaym): general extension check */ + /* for ms{,g}fi */ + if (op & VARIABLE_FLAG_MASK) + return 0; + + return have_genext() && is_s16(imm); + case SLJIT_OR32: + case SLJIT_XOR32: + case SLJIT_AND32: + /* only use if have extended immediate facility */ + /* this ensures flags are set correctly */ + return have_eimm(); + case SLJIT_AND: + case SLJIT_OR: + case SLJIT_XOR: + /* TODO(mundaym): make this more flexible */ + /* avoid using immediate variations, flags */ + /* won't be set correctly */ + return 0; + case SLJIT_ADDC32: + case SLJIT_ADDC: + /* no ADD LOGICAL WITH CARRY IMMEDIATE */ + return 0; + case SLJIT_SUB: + case SLJIT_SUB32: + case SLJIT_SUBC: + case SLJIT_SUBC32: + /* no SUBTRACT IMMEDIATE */ + /* TODO(mundaym): SUBTRACT LOGICAL IMMEDIATE */ + return 0; + } + return 0; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 dst, sljit_sw dstw, + sljit_s32 src1, sljit_sw src1w, + sljit_s32 src2, sljit_sw src2w) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op2(compiler, op, dst, dstw, src1, src1w, src2, src2w)); + ADJUST_LOCAL_OFFSET(dst, dstw); + ADJUST_LOCAL_OFFSET(src1, src1w); + ADJUST_LOCAL_OFFSET(src2, src2w); + + if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) + return SLJIT_SUCCESS; + + sljit_gpr dst_r = SLOW_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0; + + if (is_commutative(op)) { + #define SWAP_ARGS \ + do { \ + sljit_s32 t = src1; \ + sljit_sw tw = src1w; \ + src1 = src2; \ + src1w = src2w; \ + src2 = t; \ + src2w = tw; \ + } while(0); + + /* prefer immediate in src2 */ + if (src1 & SLJIT_IMM) { + SWAP_ARGS + } + + /* prefer to have src1 use same register as dst */ + if (FAST_IS_REG(src2) && gpr(src2 & REG_MASK) == dst_r) { + SWAP_ARGS + } + + /* prefer memory argument in src2 */ + if (FAST_IS_REG(src2) && (src1 & SLJIT_MEM)) { + SWAP_ARGS + } + #undef SWAP_ARGS + } + + /* src1 must be in a register */ + sljit_gpr src1_r = FAST_IS_REG(src1) ? gpr(src1 & REG_MASK) : tmp0; + if (src1 & SLJIT_IMM) + FAIL_IF(push_load_imm_inst(compiler, src1_r, src1w)); + + if (src1 & SLJIT_MEM) + FAIL_IF(load_word(compiler, src1_r, src1, src1w, tmp1, op & SLJIT_I32_OP)); + + /* emit comparison before subtract */ + if (GET_OPCODE(op) == SLJIT_SUB && (op & VARIABLE_FLAG_MASK)) { + sljit_sw cmp = 0; + switch (GET_FLAG_TYPE(op)) { + case SLJIT_LESS: + case SLJIT_LESS_EQUAL: + case SLJIT_GREATER: + case SLJIT_GREATER_EQUAL: + cmp = 1; /* unsigned */ + break; + case SLJIT_EQUAL: + case SLJIT_SIG_LESS: + case SLJIT_SIG_LESS_EQUAL: + case SLJIT_SIG_GREATER: + case SLJIT_SIG_GREATER_EQUAL: + cmp = -1; /* signed */ + break; + } + if (cmp) { + /* clear flags - no need to generate now */ + op &= ~VARIABLE_FLAG_MASK; + sljit_gpr src2_r = FAST_IS_REG(src2) ? gpr(src2 & REG_MASK) : tmp1; + if (src2 & SLJIT_IMM) { + #define LEVAL(i) i(src1_r, src2w) + if (cmp > 0 && is_u32(src2w)) { + /* unsigned */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, clfi, clgfi))); + } + else if (cmp < 0 && is_s16(src2w)) { + /* signed */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, chi, cghi))); + } + else if (cmp < 0 && is_s32(src2w)) { + /* signed */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, cfi, cgfi))); + } + #undef LEVAL + #define LEVAL(i) i(src1_r, src2_r) + else { + FAIL_IF(push_load_imm_inst(compiler, src2_r, src2w)); + if (cmp > 0) { + /* unsigned */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, clr, clgr))); + } + if (cmp < 0) { + /* signed */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, cr, cgr))); + } + } + } + else { + if (src2 & SLJIT_MEM) { + /* TODO(mundaym): comparisons with memory */ + /* load src2 into register */ + FAIL_IF(load_word(compiler, src2_r, src2, src2w, tmp1, op & SLJIT_I32_OP)); + } + if (cmp > 0) { + /* unsigned */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, clr, clgr))); + } + if (cmp < 0) { + /* signed */ + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, cr, cgr))); + } + #undef LEVAL + } + FAIL_IF(push_inst(compiler, ipm(flag_r))); + } + } + + if (!HAS_FLAGS(op) && dst == SLJIT_UNUSED) + return SLJIT_SUCCESS; + + /* need to specify signed or logical operation */ + int signed_flags = sets_signed_flag(op); + + if (is_shift(op)) { + /* handle shifts first, they have more constraints than other operations */ + sljit_sw d = 0; + sljit_gpr b = FAST_IS_REG(src2) ? gpr(src2 & REG_MASK) : r0; + if (src2 & SLJIT_IMM) + d = src2w & ((op & SLJIT_I32_OP) ? 31 : 63); + + if (src2 & SLJIT_MEM) { + /* shift amount (b) cannot be in r0 (i.e. tmp0) */ + FAIL_IF(load_word(compiler, tmp1, src2, src2w, tmp1, op & SLJIT_I32_OP)); + b = tmp1; + } + /* src1 and dst share the same register in the base 32-bit ISA */ + /* TODO(mundaym): not needed when distinct-operand facility is available */ + int workaround_alias = op & SLJIT_I32_OP && src1_r != dst_r; + if (workaround_alias) { + /* put src1 into tmp0 so we can overwrite it */ + FAIL_IF(push_inst(compiler, lr(tmp0, src1_r))); + src1_r = tmp0; + } + switch (GET_OPCODE(op) | (op & SLJIT_I32_OP)) { + case SLJIT_SHL: + FAIL_IF(push_inst(compiler, sllg(dst_r, src1_r, d, b))); + break; + case SLJIT_SHL32: + FAIL_IF(push_inst(compiler, sll(src1_r, d, b))); + break; + case SLJIT_LSHR: + FAIL_IF(push_inst(compiler, srlg(dst_r, src1_r, d, b))); + break; + case SLJIT_LSHR32: + FAIL_IF(push_inst(compiler, srl(src1_r, d, b))); + break; + case SLJIT_ASHR: + FAIL_IF(push_inst(compiler, srag(dst_r, src1_r, d, b))); + break; + case SLJIT_ASHR32: + FAIL_IF(push_inst(compiler, sra(src1_r, d, b))); + break; + default: + SLJIT_UNREACHABLE(); + } + if (workaround_alias && dst_r != src1_r) + FAIL_IF(push_inst(compiler, lr(dst_r, src1_r))); + + } + else if ((GET_OPCODE(op) == SLJIT_MUL) && HAS_FLAGS(op)) { + /* multiply instructions do not generally set flags so we need to manually */ + /* detect overflow conditions */ + /* TODO(mundaym): 64-bit overflow */ + SLJIT_ASSERT(GET_FLAG_TYPE(op) == SLJIT_MUL_OVERFLOW || + GET_FLAG_TYPE(op) == SLJIT_MUL_NOT_OVERFLOW); + sljit_gpr src2_r = FAST_IS_REG(src2) ? gpr(src2 & REG_MASK) : tmp1; + if (src2 & SLJIT_IMM) { + /* load src2 into register */ + FAIL_IF(push_load_imm_inst(compiler, src2_r, src2w)); + } + if (src2 & SLJIT_MEM) { + /* load src2 into register */ + FAIL_IF(load_word(compiler, src2_r, src2, src2w, tmp1, op & SLJIT_I32_OP)); + } + if (have_misc2()) { + #define LEVAL(i) i(dst_r, src1_r, src2_r) + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, msrkc, msgrkc))); + #undef LEVAL + } + else if (op & SLJIT_I32_OP) { + op &= ~VARIABLE_FLAG_MASK; + FAIL_IF(push_inst(compiler, lgfr(tmp0, src1_r))); + FAIL_IF(push_inst(compiler, msgfr(tmp0, src2_r))); + if (dst_r != tmp0) { + FAIL_IF(push_inst(compiler, lr(dst_r, tmp0))); + } + FAIL_IF(push_inst(compiler, aih(tmp0, 1))); + FAIL_IF(push_inst(compiler, nihf(tmp0, ~1U))); + FAIL_IF(push_inst(compiler, ipm(flag_r))); + FAIL_IF(push_inst(compiler, oilh(flag_r, 0x2000))); + } + else + return SLJIT_ERR_UNSUPPORTED; + + } + else if ((GET_OPCODE(op) == SLJIT_SUB) && (op & SLJIT_SET_Z) && !signed_flags) { + /* subtract logical instructions do not set the right flags unfortunately */ + /* instead, negate src2 and issue an add logical */ + /* TODO(mundaym): distinct operand facility where needed */ + if (src1_r != dst_r && src1_r != tmp0) { + #define LEVAL(i) i(tmp0, src1_r) + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, lr, lgr))); + src1_r = tmp0; + #undef LEVAL + } + sljit_gpr src2_r = FAST_IS_REG(src2) ? gpr(src2 & REG_MASK) : tmp1; + if (src2 & SLJIT_IMM) { + /* load src2 into register */ + FAIL_IF(push_load_imm_inst(compiler, src2_r, src2w)); + } + if (src2 & SLJIT_MEM) { + /* load src2 into register */ + FAIL_IF(load_word(compiler, src2_r, src2, src2w, tmp1, op & SLJIT_I32_OP)); + } + if (op & SLJIT_I32_OP) { + FAIL_IF(push_inst(compiler, lcr(tmp1, src2_r))); + FAIL_IF(push_inst(compiler, alr(src1_r, tmp1))); + if (src1_r != dst_r) + FAIL_IF(push_inst(compiler, lr(dst_r, src1_r))); + } + else { + FAIL_IF(push_inst(compiler, lcgr(tmp1, src2_r))); + FAIL_IF(push_inst(compiler, algr(src1_r, tmp1))); + if (src1_r != dst_r) + FAIL_IF(push_inst(compiler, lgr(dst_r, src1_r))); + } + } + else if ((src2 & SLJIT_IMM) && (src1_r == dst_r) && have_op_2_imm(op, src2w)) { + switch (GET_OPCODE(op) | (op & SLJIT_I32_OP)) { + #define LEVAL(i) i(dst_r, src2w) + case SLJIT_ADD: + if (!HAS_FLAGS(op) || signed_flags) { + FAIL_IF(push_inst(compiler, + WHEN2(is_s16(src2w), aghi, agfi))); + } + else + FAIL_IF(push_inst(compiler, LEVAL(algfi))); + + break; + case SLJIT_ADD32: + if (!HAS_FLAGS(op) || signed_flags) + FAIL_IF(push_inst(compiler, + WHEN2(is_s16(src2w), ahi, afi))); + else + FAIL_IF(push_inst(compiler, LEVAL(alfi))); + + break; + #undef LEVAL /* TODO(carenas): move down and refactor? */ + case SLJIT_MUL: + FAIL_IF(push_inst(compiler, mhi(dst_r, src2w))); + break; + case SLJIT_MUL32: + FAIL_IF(push_inst(compiler, mghi(dst_r, src2w))); + break; + case SLJIT_OR32: + FAIL_IF(push_inst(compiler, oilf(dst_r, src2w))); + break; + case SLJIT_XOR32: + FAIL_IF(push_inst(compiler, xilf(dst_r, src2w))); + break; + case SLJIT_AND32: + FAIL_IF(push_inst(compiler, nilf(dst_r, src2w))); + break; + default: + SLJIT_UNREACHABLE(); + } + } + else if ((src2 & SLJIT_IMM) && have_op_3_imm(op, src2w)) { + abort(); /* TODO(mundaym): implement */ + } + else if ((src2 & SLJIT_MEM) && (dst_r == src1_r)) { + /* most 32-bit instructions can only handle 12-bit immediate offsets */ + int need_u12 = !have_ldisp() && + (op & SLJIT_I32_OP) && + (GET_OPCODE(op) != SLJIT_ADDC) && + (GET_OPCODE(op) != SLJIT_SUBC); + struct addr mem; + if (need_u12) + FAIL_IF(make_addr_bx(compiler, &mem, src2, src2w, tmp1)); + else + FAIL_IF(make_addr_bxy(compiler, &mem, src2, src2w, tmp1)); + + int can_u12 = is_u12(mem.offset) ? 1 : 0; + sljit_ins ins = 0; + switch (GET_OPCODE(op) | (op & SLJIT_I32_OP)) { + /* 64-bit ops */ + #define LEVAL(i) EVAL(i, dst_r, mem) + case SLJIT_ADD: + ins = WHEN2(signed_flags, ag, alg); + break; + case SLJIT_SUB: + ins = WHEN2(signed_flags, sg, slg); + break; + case SLJIT_ADDC: + ins = LEVAL(alcg); + break; + case SLJIT_SUBC: + ins = LEVAL(slbg); + break; + case SLJIT_MUL: + ins = LEVAL(msg); + break; + case SLJIT_OR: + ins = LEVAL(og); + break; + case SLJIT_XOR: + ins = LEVAL(xg); + break; + case SLJIT_AND: + ins = LEVAL(ng); + break; + /* 32-bit ops */ + case SLJIT_ADD32: + if (signed_flags) + ins = WHEN2(can_u12, a, ay); + else + ins = WHEN2(can_u12, al, aly); + break; + case SLJIT_SUB32: + if (signed_flags) + ins = WHEN2(can_u12, s, sy); + else + ins = WHEN2(can_u12, sl, sly); + break; + case SLJIT_ADDC32: + ins = LEVAL(alc); + break; + case SLJIT_SUBC32: + ins = LEVAL(slb); + break; + case SLJIT_MUL32: + ins = WHEN2(can_u12, ms, msy); + break; + case SLJIT_OR32: + ins = WHEN2(can_u12, o, oy); + break; + case SLJIT_XOR32: + ins = WHEN2(can_u12, x, xy); + break; + case SLJIT_AND32: + ins = WHEN2(can_u12, n, ny); + break; + #undef LEVAL + default: + SLJIT_UNREACHABLE(); + } + FAIL_IF(push_inst(compiler, ins)); + } + else { + sljit_gpr src2_r = FAST_IS_REG(src2) ? gpr(src2 & REG_MASK) : tmp1; + if (src2 & SLJIT_IMM) { + /* load src2 into register */ + FAIL_IF(push_load_imm_inst(compiler, src2_r, src2w)); + } + if (src2 & SLJIT_MEM) { + /* load src2 into register */ + FAIL_IF(load_word(compiler, src2_r, src2, src2w, tmp1, op & SLJIT_I32_OP)); + } + /* TODO(mundaym): distinct operand facility where needed */ + #define LEVAL(i) i(tmp0, src1_r) + if (src1_r != dst_r && src1_r != tmp0) { + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, lr, lgr))); + src1_r = tmp0; + } + #undef LEVAL + sljit_ins ins = 0; + switch (GET_OPCODE(op) | (op & SLJIT_I32_OP)) { + #define LEVAL(i) i(src1_r, src2_r) + /* 64-bit ops */ + case SLJIT_ADD: + ins = WHEN2(signed_flags, agr, algr); + break; + case SLJIT_SUB: + ins = WHEN2(signed_flags, sgr, slgr); + break; + case SLJIT_ADDC: + ins = LEVAL(alcgr); + break; + case SLJIT_SUBC: + ins = LEVAL(slbgr); + break; + case SLJIT_MUL: + ins = LEVAL(msgr); + break; + case SLJIT_AND: + ins = LEVAL(ngr); + break; + case SLJIT_OR: + ins = LEVAL(ogr); + break; + case SLJIT_XOR: + ins = LEVAL(xgr); + break; + /* 32-bit ops */ + case SLJIT_ADD32: + ins = WHEN2(signed_flags, ar, alr); + break; + case SLJIT_SUB32: + ins = WHEN2(signed_flags, sr, slr); + break; + case SLJIT_ADDC32: + ins = LEVAL(alcr); + break; + case SLJIT_SUBC32: + ins = LEVAL(slbr); + break; + case SLJIT_MUL32: + ins = LEVAL(msr); + break; + case SLJIT_AND32: + ins = LEVAL(nr); + break; + case SLJIT_OR32: + ins = LEVAL(or); + break; + case SLJIT_XOR32: + ins = LEVAL(xr); + break; + #undef LEVAL + default: + SLJIT_UNREACHABLE(); + } + FAIL_IF(push_inst(compiler, ins)); + #define LEVAL(i) i(dst_r, src1_r) + if (src1_r != dst_r) + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, lr, lgr))); + #undef LEVAL + } + + /* write condition code to emulated flag register */ + if (op & VARIABLE_FLAG_MASK) + FAIL_IF(push_inst(compiler, ipm(flag_r))); + + /* write zero flag to emulated flag register */ + if (op & SLJIT_SET_Z) + FAIL_IF(push_store_zero_flag(compiler, op, dst_r)); + + /* finally write the result to memory if required */ + if (dst & SLJIT_MEM) { + SLJIT_ASSERT(dst_r != tmp1); + /* TODO(carenas): s/FAIL_IF/ return */ + FAIL_IF(store_word(compiler, dst_r, dst, dstw, tmp1, op & SLJIT_I32_OP)); + } + + return SLJIT_SUCCESS; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src( + struct sljit_compiler *compiler, + sljit_s32 op, sljit_s32 src, sljit_sw srcw) +{ + sljit_gpr src_r; + + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + src_r = FAST_IS_REG(src) ? gpr(src) : tmp1; + if (src & SLJIT_MEM) + FAIL_IF(load_word(compiler, tmp1, src, srcw, tmp1, 0)); + + return push_inst(compiler, br(src_r)); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + /* TODO(carenas): implement? */ + return SLJIT_SUCCESS; + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + /* TODO(carenas): implement */ + return SLJIT_SUCCESS; + default: + /* TODO(carenas): probably should not success by default */ + return SLJIT_SUCCESS; + } + + return SLJIT_SUCCESS; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) +{ + CHECK_REG_INDEX(check_sljit_get_register_index(reg)); + return gpr(reg); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg) +{ + CHECK_REG_INDEX(check_sljit_get_float_register_index(reg)); + abort(); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler, + void *instruction, sljit_s32 size) +{ + sljit_ins ins = 0; + + CHECK_ERROR(); + CHECK(check_sljit_emit_op_custom(compiler, instruction, size)); + + memcpy((sljit_u8 *)&ins + sizeof(ins) - size, instruction, size); + return push_inst(compiler, ins); +} + +/* --------------------------------------------------------------------- */ +/* Floating point operators */ +/* --------------------------------------------------------------------- */ + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 dst, sljit_sw dstw, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + abort(); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 dst, sljit_sw dstw, + sljit_s32 src1, sljit_sw src1w, + sljit_s32 src2, sljit_sw src2w) +{ + CHECK_ERROR(); + abort(); +} + +/* --------------------------------------------------------------------- */ +/* Other instructions */ +/* --------------------------------------------------------------------- */ + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + if (FAST_IS_REG(dst)) + return push_inst(compiler, lgr(gpr(dst), fast_link_r)); + + /* memory */ + return store_word(compiler, fast_link_r, dst, dstw, tmp1, 0); +} + +/* --------------------------------------------------------------------- */ +/* Conditional instructions */ +/* --------------------------------------------------------------------- */ + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler) +{ + struct sljit_label *label; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_label(compiler)); + + if (compiler->last_label && compiler->last_label->size == compiler->size) + return compiler->last_label; + + label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label)); + PTR_FAIL_IF(!label); + set_label(label, compiler); + return label; +} + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type) +{ + sljit_u8 mask = ((type & 0xff) < SLJIT_JUMP) ? get_cc(type & 0xff) : 0xf; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_jump(compiler, type)); + + /* reload condition code */ + if (mask != 0xf) + PTR_FAIL_IF(push_load_cc(compiler, type & 0xff)); + + /* record jump */ + struct sljit_jump *jump = (struct sljit_jump *) + ensure_abuf(compiler, sizeof(struct sljit_jump)); + PTR_FAIL_IF(!jump); + set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP); + jump->addr = compiler->size; + + /* emit jump instruction */ + type &= 0xff; + if (type >= SLJIT_FAST_CALL) + PTR_FAIL_IF(push_inst(compiler, brasl(type == SLJIT_FAST_CALL ? fast_link_r : link_r, 0))); + else + PTR_FAIL_IF(push_inst(compiler, brcl(mask, 0))); + + return jump; +} + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type, + sljit_s32 arg_types) +{ + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types)); + +#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \ + || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) + compiler->skip_checks = 1; +#endif + + return sljit_emit_jump(compiler, type); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw) +{ + sljit_gpr src_r = FAST_IS_REG(src) ? gpr(src) : tmp1; + + CHECK_ERROR(); + CHECK(check_sljit_emit_ijump(compiler, type, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + if (src & SLJIT_IMM) { + SLJIT_ASSERT(!(srcw & 1)); /* target address must be even */ + FAIL_IF(push_load_imm_inst(compiler, src_r, srcw)); + } + else if (src & SLJIT_MEM) + FAIL_IF(load_word(compiler, src_r, src, srcw, tmp1, 0 /* 64-bit */)); + + /* emit jump instruction */ + if (type >= SLJIT_FAST_CALL) + return push_inst(compiler, basr(type == SLJIT_FAST_CALL ? fast_link_r : link_r, src_r)); + + return push_inst(compiler, br(src_r)); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type, + sljit_s32 arg_types, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw)); + +#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \ + || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) + compiler->skip_checks = 1; +#endif + + return sljit_emit_ijump(compiler, type, src, srcw); +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 dst, sljit_sw dstw, + sljit_s32 type) +{ + sljit_u8 mask = get_cc(type & 0xff); + + CHECK_ERROR(); + CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type)); + + sljit_gpr dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0; + sljit_gpr loc_r = tmp1; + switch (GET_OPCODE(op)) { + case SLJIT_AND: + case SLJIT_OR: + case SLJIT_XOR: + /* dst is also source operand */ + if (dst & SLJIT_MEM) + FAIL_IF(load_word(compiler, dst_r, dst, dstw, tmp1, op & SLJIT_I32_OP)); + + break; + case SLJIT_MOV: + case (SLJIT_MOV32 & ~SLJIT_I32_OP): + /* can write straight into destination */ + loc_r = dst_r; + break; + default: + SLJIT_UNREACHABLE(); + } + + if (mask != 0xf) + FAIL_IF(push_load_cc(compiler, type & 0xff)); + + /* TODO(mundaym): fold into cmov helper function? */ + #define LEVAL(i) i(loc_r, 1, mask) + if (have_lscond2()) { + FAIL_IF(push_load_imm_inst(compiler, loc_r, 0)); + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, lochi, locghi))); + } else { + /* TODO(mundaym): no load/store-on-condition 2 facility (ipm? branch-and-set?) */ + abort(); + } + #undef LEVAL + + /* apply bitwise op and set condition codes */ + switch (GET_OPCODE(op)) { + #define LEVAL(i) i(dst_r, loc_r) + case SLJIT_AND: + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, nr, ngr))); + break; + case SLJIT_OR: + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, or, ogr))); + break; + case SLJIT_XOR: + FAIL_IF(push_inst(compiler, + WHEN2(op & SLJIT_I32_OP, xr, xgr))); + break; + #undef LEVAL + } + + /* set zero flag if needed */ + if (op & SLJIT_SET_Z) + FAIL_IF(push_store_zero_flag(compiler, op, dst_r)); + + /* store result to memory if required */ + /* TODO(carenas): s/FAIL_IF/ return */ + if (dst & SLJIT_MEM) + FAIL_IF(store_word(compiler, dst_r, dst, dstw, tmp1, op & SLJIT_I32_OP)); + + return SLJIT_SUCCESS; +} + +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type, + sljit_s32 dst_reg, + sljit_s32 src, sljit_sw srcw) +{ + sljit_u8 mask = get_cc(type & 0xff); + sljit_gpr dst_r = gpr(dst_reg & ~SLJIT_I32_OP); + sljit_gpr src_r = FAST_IS_REG(src) ? gpr(src) : tmp0; + + CHECK_ERROR(); + CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw)); + + if (mask != 0xf) + FAIL_IF(push_load_cc(compiler, type & 0xff)); + + if (src & SLJIT_IMM) { + /* TODO(mundaym): fast path with lscond2 */ + FAIL_IF(push_load_imm_inst(compiler, src_r, srcw)); + } + + #define LEVAL(i) i(dst_r, src_r, mask) + if (have_lscond1()) + return push_inst(compiler, + WHEN2(dst_reg & SLJIT_I32_OP, locr, locgr)); + + #undef LEVAL + + /* TODO(mundaym): implement */ + return SLJIT_ERR_UNSUPPORTED; +} + +/* --------------------------------------------------------------------- */ +/* Other instructions */ +/* --------------------------------------------------------------------- */ + +/* On s390x we build a literal pool to hold constants. This has two main + advantages: + + 1. we only need one instruction in the instruction stream (LGRL) + 2. we can store 64 bit addresses and use 32 bit offsets + + To retrofit the extra information needed to build the literal pool we + add a new sljit_s390x_const struct that contains the initial value but + can still be cast to a sljit_const. */ + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) +{ + struct sljit_s390x_const *const_; + sljit_gpr dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); + + const_ = (struct sljit_s390x_const*)ensure_abuf(compiler, + sizeof(struct sljit_s390x_const)); + PTR_FAIL_IF(!const_); + set_const((struct sljit_const*)const_, compiler); + const_->init_value = init_value; + + dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0; + if (have_genext()) + PTR_FAIL_IF(push_inst(compiler, sljit_ins_const | lgrl(dst_r, 0))); + else { + PTR_FAIL_IF(push_inst(compiler, sljit_ins_const | larl(tmp1, 0))); + PTR_FAIL_IF(push_inst(compiler, lg(dst_r, 0, r0, tmp1))); + } + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(store_word(compiler, dst_r, dst, dstw, tmp1, 0 /* always 64-bit */)); + + return (struct sljit_const*)const_; +} + +SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) +{ + /* Update the constant pool. */ + sljit_uw *ptr = (sljit_uw *)addr; + SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0); + *ptr = new_target; + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1); + SLJIT_CACHE_FLUSH(ptr, ptr + 1); +} + +SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) +{ + sljit_set_jump_addr(addr, new_constant, executable_offset); +} + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label *sljit_emit_put_label( + struct sljit_compiler *compiler, + sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_gpr dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + + dst_r = FAST_IS_REG(dst) ? gpr(dst & REG_MASK) : tmp0; + + if (have_genext()) + PTR_FAIL_IF(push_inst(compiler, lgrl(dst_r, 0))); + else { + PTR_FAIL_IF(push_inst(compiler, larl(tmp1, 0))); + PTR_FAIL_IF(push_inst(compiler, lg(dst_r, 0, r0, tmp1))); + } + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(store_word(compiler, dst_r, dst, dstw, tmp1, 0)); + + return put_label; +} + +/* TODO(carenas): EVAL probably should move up or be refactored */ +#undef WHEN2 +#undef EVAL + +#undef tmp1 +#undef tmp0 + +/* TODO(carenas): undef other macros that spill like is_u12? */ diff --git a/dist2/src/sljit/sljitNativeSPARC_32.c b/dist2/src/sljit/sljitNativeSPARC_32.c index 0671b130..e5167f02 100644 --- a/dist2/src/sljit/sljitNativeSPARC_32.c +++ b/dist2/src/sljit/sljitNativeSPARC_32.c @@ -266,19 +266,18 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + SLJIT_ASSERT(((inst[0] & 0xc1c00000) == 0x01000000) && ((inst[1] & 0xc1f82000) == 0x80102000)); inst[0] = (inst[0] & 0xffc00000) | ((new_target >> 10) & 0x3fffff); inst[1] = (inst[1] & 0xfffffc00) | (new_target & 0x3ff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & 0xffc00000) | ((new_constant >> 10) & 0x3fffff); - inst[1] = (inst[1] & 0xfffffc00) | (new_constant & 0x3ff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/dist2/src/sljit/sljitNativeSPARC_common.c b/dist2/src/sljit/sljitNativeSPARC_common.c index 669ecd81..544d80d0 100644 --- a/dist2/src/sljit/sljitNativeSPARC_common.c +++ b/dist2/src/sljit/sljitNativeSPARC_common.c @@ -298,57 +298,71 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil sljit_ins *buf_ptr; sljit_ins *buf_end; sljit_uw word_count; + sljit_uw next_addr; sljit_sw executable_offset; sljit_uw addr; struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; code_ptr = code; word_count = 0; + next_addr = 0; executable_offset = SLJIT_EXEC_OFFSET(code); label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; do { buf_ptr = (sljit_ins*)buf->memory; buf_end = buf_ptr + (buf->used_size >> 2); do { *code_ptr = *buf_ptr++; - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - /* These structures are ordered by their address. */ - if (label && label->size == word_count) { - /* Just recording the address. */ - label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); - label->size = code_ptr - code; - label = label->next; - } - if (jump && jump->addr == word_count) { + if (next_addr == word_count) { + SLJIT_ASSERT(!label || label->size >= word_count); + SLJIT_ASSERT(!jump || jump->addr >= word_count); + SLJIT_ASSERT(!const_ || const_->addr >= word_count); + SLJIT_ASSERT(!put_label || put_label->addr >= word_count); + + /* These structures are ordered by their address. */ + if (label && label->size == word_count) { + /* Just recording the address. */ + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + label->size = code_ptr - code; + label = label->next; + } + if (jump && jump->addr == word_count) { #if (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) - jump->addr = (sljit_uw)(code_ptr - 3); + jump->addr = (sljit_uw)(code_ptr - 3); #else - jump->addr = (sljit_uw)(code_ptr - 6); + jump->addr = (sljit_uw)(code_ptr - 6); #endif - code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset); - jump = jump->next; - } - if (const_ && const_->addr == word_count) { - /* Just recording the address. */ - const_->addr = (sljit_uw)code_ptr; - const_ = const_->next; + code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset); + jump = jump->next; + } + if (const_ && const_->addr == word_count) { + /* Just recording the address. */ + const_->addr = (sljit_uw)code_ptr; + const_ = const_->next; + } + if (put_label && put_label->addr == word_count) { + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; + put_label = put_label->next; + } + next_addr = compute_next_addr(label, jump, const_, put_label); } code_ptr ++; word_count ++; @@ -366,6 +380,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); SLJIT_ASSERT(code_ptr - code <= (sljit_s32)compiler->size); jump = compiler->jumps; @@ -389,8 +404,9 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil /* Set the fields of immediate loads. */ #if (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) - buf_ptr[0] = (buf_ptr[0] & 0xffc00000) | ((addr >> 10) & 0x3fffff); - buf_ptr[1] = (buf_ptr[1] & 0xfffffc00) | (addr & 0x3ff); + SLJIT_ASSERT(((buf_ptr[0] & 0xc1cfffff) == 0x01000000) && ((buf_ptr[1] & 0xc1f83fff) == 0x80102000)); + buf_ptr[0] |= (addr >> 10) & 0x3fffff; + buf_ptr[1] |= addr & 0x3ff; #else #error "Implementation required" #endif @@ -398,6 +414,20 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } + put_label = compiler->put_labels; + while (put_label) { + addr = put_label->label->addr; + buf_ptr = (sljit_ins *)put_label->addr; + +#if (defined SLJIT_CONFIG_SPARC_32 && SLJIT_CONFIG_SPARC_32) + SLJIT_ASSERT(((buf_ptr[0] & 0xc1cfffff) == 0x01000000) && ((buf_ptr[1] & 0xc1f83fff) == 0x80102000)); + buf_ptr[0] |= (addr >> 10) & 0x3fffff; + buf_ptr[1] |= addr & 0x3ff; +#else +#error "Implementation required" +#endif + put_label = put_label->next; + } compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; @@ -407,6 +437,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -421,6 +452,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) return 1; #endif + case SLJIT_HAS_ZERO_REGISTER: + return 1; + #if (defined SLJIT_CONFIG_SPARC_64 && SLJIT_CONFIG_SPARC_64) case SLJIT_HAS_CMOV: return 1; @@ -842,6 +876,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile #else #error "Implementation required" #endif + case SLJIT_ENDBR: + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return SLJIT_SUCCESS; } return SLJIT_SUCCESS; @@ -858,9 +895,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile ADJUST_LOCAL_OFFSET(dst, dstw); ADJUST_LOCAL_OFFSET(src, srcw); - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) - return SLJIT_SUCCESS; - op = GET_OPCODE(op); switch (op) { case SLJIT_MOV: @@ -941,6 +975,33 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_SUCCESS; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + switch (op) { + case SLJIT_FAST_RETURN: + if (FAST_IS_REG(src)) + FAIL_IF(push_inst(compiler, OR | D(TMP_LINK) | S1(0) | S2(src), DR(TMP_LINK))); + else + FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_LINK, src, srcw)); + + FAIL_IF(push_inst(compiler, JMPL | D(0) | S1(TMP_LINK) | IMM(8), UNMOVABLE_INS)); + return push_inst(compiler, NOP, UNMOVABLE_INS); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + return SLJIT_SUCCESS; + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -1185,25 +1246,12 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * ADJUST_LOCAL_OFFSET(dst, dstw); if (FAST_IS_REG(dst)) - return push_inst(compiler, OR | D(dst) | S1(0) | S2(TMP_LINK), DR(dst)); + return push_inst(compiler, OR | D(dst) | S1(0) | S2(TMP_LINK), UNMOVABLE_INS); /* Memory. */ - return emit_op_mem(compiler, WORD_DATA, TMP_LINK, dst, dstw); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) - FAIL_IF(push_inst(compiler, OR | D(TMP_LINK) | S1(0) | S2(src), DR(TMP_LINK))); - else - FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_LINK, src, srcw)); - - FAIL_IF(push_inst(compiler, JMPL | D(0) | S1(TMP_LINK) | IMM(8), UNMOVABLE_INS)); - return push_inst(compiler, NOP, UNMOVABLE_INS); + FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_LINK, dst, dstw)); + compiler->delay_slot = UNMOVABLE_INS; + return SLJIT_SUCCESS; } /* --------------------------------------------------------------------- */ @@ -1465,8 +1513,8 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compil SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) { - sljit_s32 reg; struct sljit_const *const_; + sljit_s32 dst_r; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); @@ -1476,11 +1524,31 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi PTR_FAIL_IF(!const_); set_const(const_, compiler); - reg = FAST_IS_REG(dst) ? dst : TMP_REG2; - - PTR_FAIL_IF(emit_const(compiler, reg, init_value)); + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; + PTR_FAIL_IF(emit_const(compiler, dst_r, init_value)); if (dst & SLJIT_MEM) PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw)); return const_; } + +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_s32 dst_r; + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + + dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2; + PTR_FAIL_IF(emit_const(compiler, dst_r, 0)); + + if (dst & SLJIT_MEM) + PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw)); + return put_label; +} diff --git a/dist2/src/sljit/sljitNativeTILEGX-encoder.c b/dist2/src/sljit/sljitNativeTILEGX-encoder.c deleted file mode 100644 index dd82ebae..00000000 --- a/dist2/src/sljit/sljitNativeTILEGX-encoder.c +++ /dev/null @@ -1,10159 +0,0 @@ -/* - * Stack-less Just-In-Time compiler - * - * Copyright 2013-2013 Tilera Corporation(jiwang@tilera.com). All rights reserved. - * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are - * permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, this list - * of conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* This code is owned by Tilera Corporation, and distributed as part - of multiple projects. In sljit, the code is under BSD licence. */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#define BFD_RELOC(x) R_##x - -/* Special registers. */ -#define TREG_LR 55 -#define TREG_SN 56 -#define TREG_ZERO 63 - -/* Canonical name of each register. */ -const char *const tilegx_register_names[] = -{ - "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", - "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", - "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", - "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31", - "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39", - "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47", - "r48", "r49", "r50", "r51", "r52", "tp", "sp", "lr", - "sn", "idn0", "idn1", "udn0", "udn1", "udn2", "udn3", "zero" -}; - -enum -{ - R_NONE = 0, - R_TILEGX_NONE = 0, - R_TILEGX_64 = 1, - R_TILEGX_32 = 2, - R_TILEGX_16 = 3, - R_TILEGX_8 = 4, - R_TILEGX_64_PCREL = 5, - R_TILEGX_32_PCREL = 6, - R_TILEGX_16_PCREL = 7, - R_TILEGX_8_PCREL = 8, - R_TILEGX_HW0 = 9, - R_TILEGX_HW1 = 10, - R_TILEGX_HW2 = 11, - R_TILEGX_HW3 = 12, - R_TILEGX_HW0_LAST = 13, - R_TILEGX_HW1_LAST = 14, - R_TILEGX_HW2_LAST = 15, - R_TILEGX_COPY = 16, - R_TILEGX_GLOB_DAT = 17, - R_TILEGX_JMP_SLOT = 18, - R_TILEGX_RELATIVE = 19, - R_TILEGX_BROFF_X1 = 20, - R_TILEGX_JUMPOFF_X1 = 21, - R_TILEGX_JUMPOFF_X1_PLT = 22, - R_TILEGX_IMM8_X0 = 23, - R_TILEGX_IMM8_Y0 = 24, - R_TILEGX_IMM8_X1 = 25, - R_TILEGX_IMM8_Y1 = 26, - R_TILEGX_DEST_IMM8_X1 = 27, - R_TILEGX_MT_IMM14_X1 = 28, - R_TILEGX_MF_IMM14_X1 = 29, - R_TILEGX_MMSTART_X0 = 30, - R_TILEGX_MMEND_X0 = 31, - R_TILEGX_SHAMT_X0 = 32, - R_TILEGX_SHAMT_X1 = 33, - R_TILEGX_SHAMT_Y0 = 34, - R_TILEGX_SHAMT_Y1 = 35, - R_TILEGX_IMM16_X0_HW0 = 36, - R_TILEGX_IMM16_X1_HW0 = 37, - R_TILEGX_IMM16_X0_HW1 = 38, - R_TILEGX_IMM16_X1_HW1 = 39, - R_TILEGX_IMM16_X0_HW2 = 40, - R_TILEGX_IMM16_X1_HW2 = 41, - R_TILEGX_IMM16_X0_HW3 = 42, - R_TILEGX_IMM16_X1_HW3 = 43, - R_TILEGX_IMM16_X0_HW0_LAST = 44, - R_TILEGX_IMM16_X1_HW0_LAST = 45, - R_TILEGX_IMM16_X0_HW1_LAST = 46, - R_TILEGX_IMM16_X1_HW1_LAST = 47, - R_TILEGX_IMM16_X0_HW2_LAST = 48, - R_TILEGX_IMM16_X1_HW2_LAST = 49, - R_TILEGX_IMM16_X0_HW0_PCREL = 50, - R_TILEGX_IMM16_X1_HW0_PCREL = 51, - R_TILEGX_IMM16_X0_HW1_PCREL = 52, - R_TILEGX_IMM16_X1_HW1_PCREL = 53, - R_TILEGX_IMM16_X0_HW2_PCREL = 54, - R_TILEGX_IMM16_X1_HW2_PCREL = 55, - R_TILEGX_IMM16_X0_HW3_PCREL = 56, - R_TILEGX_IMM16_X1_HW3_PCREL = 57, - R_TILEGX_IMM16_X0_HW0_LAST_PCREL = 58, - R_TILEGX_IMM16_X1_HW0_LAST_PCREL = 59, - R_TILEGX_IMM16_X0_HW1_LAST_PCREL = 60, - R_TILEGX_IMM16_X1_HW1_LAST_PCREL = 61, - R_TILEGX_IMM16_X0_HW2_LAST_PCREL = 62, - R_TILEGX_IMM16_X1_HW2_LAST_PCREL = 63, - R_TILEGX_IMM16_X0_HW0_GOT = 64, - R_TILEGX_IMM16_X1_HW0_GOT = 65, - - R_TILEGX_IMM16_X0_HW0_PLT_PCREL = 66, - R_TILEGX_IMM16_X1_HW0_PLT_PCREL = 67, - R_TILEGX_IMM16_X0_HW1_PLT_PCREL = 68, - R_TILEGX_IMM16_X1_HW1_PLT_PCREL = 69, - R_TILEGX_IMM16_X0_HW2_PLT_PCREL = 70, - R_TILEGX_IMM16_X1_HW2_PLT_PCREL = 71, - - R_TILEGX_IMM16_X0_HW0_LAST_GOT = 72, - R_TILEGX_IMM16_X1_HW0_LAST_GOT = 73, - R_TILEGX_IMM16_X0_HW1_LAST_GOT = 74, - R_TILEGX_IMM16_X1_HW1_LAST_GOT = 75, - R_TILEGX_IMM16_X0_HW0_TLS_GD = 78, - R_TILEGX_IMM16_X1_HW0_TLS_GD = 79, - R_TILEGX_IMM16_X0_HW0_TLS_LE = 80, - R_TILEGX_IMM16_X1_HW0_TLS_LE = 81, - R_TILEGX_IMM16_X0_HW0_LAST_TLS_LE = 82, - R_TILEGX_IMM16_X1_HW0_LAST_TLS_LE = 83, - R_TILEGX_IMM16_X0_HW1_LAST_TLS_LE = 84, - R_TILEGX_IMM16_X1_HW1_LAST_TLS_LE = 85, - R_TILEGX_IMM16_X0_HW0_LAST_TLS_GD = 86, - R_TILEGX_IMM16_X1_HW0_LAST_TLS_GD = 87, - R_TILEGX_IMM16_X0_HW1_LAST_TLS_GD = 88, - R_TILEGX_IMM16_X1_HW1_LAST_TLS_GD = 89, - R_TILEGX_IMM16_X0_HW0_TLS_IE = 92, - R_TILEGX_IMM16_X1_HW0_TLS_IE = 93, - - R_TILEGX_IMM16_X0_HW0_LAST_PLT_PCREL = 94, - R_TILEGX_IMM16_X1_HW0_LAST_PLT_PCREL = 95, - R_TILEGX_IMM16_X0_HW1_LAST_PLT_PCREL = 96, - R_TILEGX_IMM16_X1_HW1_LAST_PLT_PCREL = 97, - R_TILEGX_IMM16_X0_HW2_LAST_PLT_PCREL = 98, - R_TILEGX_IMM16_X1_HW2_LAST_PLT_PCREL = 99, - - R_TILEGX_IMM16_X0_HW0_LAST_TLS_IE = 100, - R_TILEGX_IMM16_X1_HW0_LAST_TLS_IE = 101, - R_TILEGX_IMM16_X0_HW1_LAST_TLS_IE = 102, - R_TILEGX_IMM16_X1_HW1_LAST_TLS_IE = 103, - R_TILEGX_TLS_DTPMOD64 = 106, - R_TILEGX_TLS_DTPOFF64 = 107, - R_TILEGX_TLS_TPOFF64 = 108, - R_TILEGX_TLS_DTPMOD32 = 109, - R_TILEGX_TLS_DTPOFF32 = 110, - R_TILEGX_TLS_TPOFF32 = 111, - R_TILEGX_TLS_GD_CALL = 112, - R_TILEGX_IMM8_X0_TLS_GD_ADD = 113, - R_TILEGX_IMM8_X1_TLS_GD_ADD = 114, - R_TILEGX_IMM8_Y0_TLS_GD_ADD = 115, - R_TILEGX_IMM8_Y1_TLS_GD_ADD = 116, - R_TILEGX_TLS_IE_LOAD = 117, - R_TILEGX_IMM8_X0_TLS_ADD = 118, - R_TILEGX_IMM8_X1_TLS_ADD = 119, - R_TILEGX_IMM8_Y0_TLS_ADD = 120, - R_TILEGX_IMM8_Y1_TLS_ADD = 121, - R_TILEGX_GNU_VTINHERIT = 128, - R_TILEGX_GNU_VTENTRY = 129, - R_TILEGX_IRELATIVE = 130, - R_TILEGX_NUM = 131 -}; - -typedef enum -{ - TILEGX_PIPELINE_X0, - TILEGX_PIPELINE_X1, - TILEGX_PIPELINE_Y0, - TILEGX_PIPELINE_Y1, - TILEGX_PIPELINE_Y2, -} tilegx_pipeline; - -typedef unsigned long long tilegx_bundle_bits; - -/* These are the bits that determine if a bundle is in the X encoding. */ -#define TILEGX_BUNDLE_MODE_MASK ((tilegx_bundle_bits)3 << 62) - -enum -{ - /* Maximum number of instructions in a bundle (2 for X, 3 for Y). */ - TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE = 3, - - /* How many different pipeline encodings are there? X0, X1, Y0, Y1, Y2. */ - TILEGX_NUM_PIPELINE_ENCODINGS = 5, - - /* Log base 2 of TILEGX_BUNDLE_SIZE_IN_BYTES. */ - TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES = 3, - - /* Instructions take this many bytes. */ - TILEGX_BUNDLE_SIZE_IN_BYTES = 1 << TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES, - - /* Log base 2 of TILEGX_BUNDLE_ALIGNMENT_IN_BYTES. */ - TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES = 3, - - /* Bundles should be aligned modulo this number of bytes. */ - TILEGX_BUNDLE_ALIGNMENT_IN_BYTES = - (1 << TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES), - - /* Number of registers (some are magic, such as network I/O). */ - TILEGX_NUM_REGISTERS = 64, -}; - -/* Make a few "tile_" variables to simplify common code between - architectures. */ - -typedef tilegx_bundle_bits tile_bundle_bits; -#define TILE_BUNDLE_SIZE_IN_BYTES TILEGX_BUNDLE_SIZE_IN_BYTES -#define TILE_BUNDLE_ALIGNMENT_IN_BYTES TILEGX_BUNDLE_ALIGNMENT_IN_BYTES -#define TILE_LOG2_BUNDLE_ALIGNMENT_IN_BYTES \ - TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES - -/* 64-bit pattern for a { bpt ; nop } bundle. */ -#define TILEGX_BPT_BUNDLE 0x286a44ae51485000ULL - -typedef enum -{ - TILEGX_OP_TYPE_REGISTER, - TILEGX_OP_TYPE_IMMEDIATE, - TILEGX_OP_TYPE_ADDRESS, - TILEGX_OP_TYPE_SPR -} tilegx_operand_type; - -struct tilegx_operand -{ - /* Is this operand a register, immediate or address? */ - tilegx_operand_type type; - - /* The default relocation type for this operand. */ - signed int default_reloc : 16; - - /* How many bits is this value? (used for range checking) */ - unsigned int num_bits : 5; - - /* Is the value signed? (used for range checking) */ - unsigned int is_signed : 1; - - /* Is this operand a source register? */ - unsigned int is_src_reg : 1; - - /* Is this operand written? (i.e. is it a destination register) */ - unsigned int is_dest_reg : 1; - - /* Is this operand PC-relative? */ - unsigned int is_pc_relative : 1; - - /* By how many bits do we right shift the value before inserting? */ - unsigned int rightshift : 2; - - /* Return the bits for this operand to be ORed into an existing bundle. */ - tilegx_bundle_bits (*insert) (int op); - - /* Extract this operand and return it. */ - unsigned int (*extract) (tilegx_bundle_bits bundle); -}; - -typedef enum -{ - TILEGX_OPC_BPT, - TILEGX_OPC_INFO, - TILEGX_OPC_INFOL, - TILEGX_OPC_LD4S_TLS, - TILEGX_OPC_LD_TLS, - TILEGX_OPC_MOVE, - TILEGX_OPC_MOVEI, - TILEGX_OPC_MOVELI, - TILEGX_OPC_PREFETCH, - TILEGX_OPC_PREFETCH_ADD_L1, - TILEGX_OPC_PREFETCH_ADD_L1_FAULT, - TILEGX_OPC_PREFETCH_ADD_L2, - TILEGX_OPC_PREFETCH_ADD_L2_FAULT, - TILEGX_OPC_PREFETCH_ADD_L3, - TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - TILEGX_OPC_PREFETCH_L1, - TILEGX_OPC_PREFETCH_L1_FAULT, - TILEGX_OPC_PREFETCH_L2, - TILEGX_OPC_PREFETCH_L2_FAULT, - TILEGX_OPC_PREFETCH_L3, - TILEGX_OPC_PREFETCH_L3_FAULT, - TILEGX_OPC_RAISE, - TILEGX_OPC_ADD, - TILEGX_OPC_ADDI, - TILEGX_OPC_ADDLI, - TILEGX_OPC_ADDX, - TILEGX_OPC_ADDXI, - TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXSC, - TILEGX_OPC_AND, - TILEGX_OPC_ANDI, - TILEGX_OPC_BEQZ, - TILEGX_OPC_BEQZT, - TILEGX_OPC_BFEXTS, - TILEGX_OPC_BFEXTU, - TILEGX_OPC_BFINS, - TILEGX_OPC_BGEZ, - TILEGX_OPC_BGEZT, - TILEGX_OPC_BGTZ, - TILEGX_OPC_BGTZT, - TILEGX_OPC_BLBC, - TILEGX_OPC_BLBCT, - TILEGX_OPC_BLBS, - TILEGX_OPC_BLBST, - TILEGX_OPC_BLEZ, - TILEGX_OPC_BLEZT, - TILEGX_OPC_BLTZ, - TILEGX_OPC_BLTZT, - TILEGX_OPC_BNEZ, - TILEGX_OPC_BNEZT, - TILEGX_OPC_CLZ, - TILEGX_OPC_CMOVEQZ, - TILEGX_OPC_CMOVNEZ, - TILEGX_OPC_CMPEQ, - TILEGX_OPC_CMPEQI, - TILEGX_OPC_CMPEXCH, - TILEGX_OPC_CMPEXCH4, - TILEGX_OPC_CMPLES, - TILEGX_OPC_CMPLEU, - TILEGX_OPC_CMPLTS, - TILEGX_OPC_CMPLTSI, - TILEGX_OPC_CMPLTU, - TILEGX_OPC_CMPLTUI, - TILEGX_OPC_CMPNE, - TILEGX_OPC_CMUL, - TILEGX_OPC_CMULA, - TILEGX_OPC_CMULAF, - TILEGX_OPC_CMULF, - TILEGX_OPC_CMULFR, - TILEGX_OPC_CMULH, - TILEGX_OPC_CMULHR, - TILEGX_OPC_CRC32_32, - TILEGX_OPC_CRC32_8, - TILEGX_OPC_CTZ, - TILEGX_OPC_DBLALIGN, - TILEGX_OPC_DBLALIGN2, - TILEGX_OPC_DBLALIGN4, - TILEGX_OPC_DBLALIGN6, - TILEGX_OPC_DRAIN, - TILEGX_OPC_DTLBPR, - TILEGX_OPC_EXCH, - TILEGX_OPC_EXCH4, - TILEGX_OPC_FDOUBLE_ADD_FLAGS, - TILEGX_OPC_FDOUBLE_ADDSUB, - TILEGX_OPC_FDOUBLE_MUL_FLAGS, - TILEGX_OPC_FDOUBLE_PACK1, - TILEGX_OPC_FDOUBLE_PACK2, - TILEGX_OPC_FDOUBLE_SUB_FLAGS, - TILEGX_OPC_FDOUBLE_UNPACK_MAX, - TILEGX_OPC_FDOUBLE_UNPACK_MIN, - TILEGX_OPC_FETCHADD, - TILEGX_OPC_FETCHADD4, - TILEGX_OPC_FETCHADDGEZ, - TILEGX_OPC_FETCHADDGEZ4, - TILEGX_OPC_FETCHAND, - TILEGX_OPC_FETCHAND4, - TILEGX_OPC_FETCHOR, - TILEGX_OPC_FETCHOR4, - TILEGX_OPC_FINV, - TILEGX_OPC_FLUSH, - TILEGX_OPC_FLUSHWB, - TILEGX_OPC_FNOP, - TILEGX_OPC_FSINGLE_ADD1, - TILEGX_OPC_FSINGLE_ADDSUB2, - TILEGX_OPC_FSINGLE_MUL1, - TILEGX_OPC_FSINGLE_MUL2, - TILEGX_OPC_FSINGLE_PACK1, - TILEGX_OPC_FSINGLE_PACK2, - TILEGX_OPC_FSINGLE_SUB1, - TILEGX_OPC_ICOH, - TILEGX_OPC_ILL, - TILEGX_OPC_INV, - TILEGX_OPC_IRET, - TILEGX_OPC_J, - TILEGX_OPC_JAL, - TILEGX_OPC_JALR, - TILEGX_OPC_JALRP, - TILEGX_OPC_JR, - TILEGX_OPC_JRP, - TILEGX_OPC_LD, - TILEGX_OPC_LD1S, - TILEGX_OPC_LD1S_ADD, - TILEGX_OPC_LD1U, - TILEGX_OPC_LD1U_ADD, - TILEGX_OPC_LD2S, - TILEGX_OPC_LD2S_ADD, - TILEGX_OPC_LD2U, - TILEGX_OPC_LD2U_ADD, - TILEGX_OPC_LD4S, - TILEGX_OPC_LD4S_ADD, - TILEGX_OPC_LD4U, - TILEGX_OPC_LD4U_ADD, - TILEGX_OPC_LD_ADD, - TILEGX_OPC_LDNA, - TILEGX_OPC_LDNA_ADD, - TILEGX_OPC_LDNT, - TILEGX_OPC_LDNT1S, - TILEGX_OPC_LDNT1S_ADD, - TILEGX_OPC_LDNT1U, - TILEGX_OPC_LDNT1U_ADD, - TILEGX_OPC_LDNT2S, - TILEGX_OPC_LDNT2S_ADD, - TILEGX_OPC_LDNT2U, - TILEGX_OPC_LDNT2U_ADD, - TILEGX_OPC_LDNT4S, - TILEGX_OPC_LDNT4S_ADD, - TILEGX_OPC_LDNT4U, - TILEGX_OPC_LDNT4U_ADD, - TILEGX_OPC_LDNT_ADD, - TILEGX_OPC_LNK, - TILEGX_OPC_MF, - TILEGX_OPC_MFSPR, - TILEGX_OPC_MM, - TILEGX_OPC_MNZ, - TILEGX_OPC_MTSPR, - TILEGX_OPC_MUL_HS_HS, - TILEGX_OPC_MUL_HS_HU, - TILEGX_OPC_MUL_HS_LS, - TILEGX_OPC_MUL_HS_LU, - TILEGX_OPC_MUL_HU_HU, - TILEGX_OPC_MUL_HU_LS, - TILEGX_OPC_MUL_HU_LU, - TILEGX_OPC_MUL_LS_LS, - TILEGX_OPC_MUL_LS_LU, - TILEGX_OPC_MUL_LU_LU, - TILEGX_OPC_MULA_HS_HS, - TILEGX_OPC_MULA_HS_HU, - TILEGX_OPC_MULA_HS_LS, - TILEGX_OPC_MULA_HS_LU, - TILEGX_OPC_MULA_HU_HU, - TILEGX_OPC_MULA_HU_LS, - TILEGX_OPC_MULA_HU_LU, - TILEGX_OPC_MULA_LS_LS, - TILEGX_OPC_MULA_LS_LU, - TILEGX_OPC_MULA_LU_LU, - TILEGX_OPC_MULAX, - TILEGX_OPC_MULX, - TILEGX_OPC_MZ, - TILEGX_OPC_NAP, - TILEGX_OPC_NOP, - TILEGX_OPC_NOR, - TILEGX_OPC_OR, - TILEGX_OPC_ORI, - TILEGX_OPC_PCNT, - TILEGX_OPC_REVBITS, - TILEGX_OPC_REVBYTES, - TILEGX_OPC_ROTL, - TILEGX_OPC_ROTLI, - TILEGX_OPC_SHL, - TILEGX_OPC_SHL16INSLI, - TILEGX_OPC_SHL1ADD, - TILEGX_OPC_SHL1ADDX, - TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL2ADDX, - TILEGX_OPC_SHL3ADD, - TILEGX_OPC_SHL3ADDX, - TILEGX_OPC_SHLI, - TILEGX_OPC_SHLX, - TILEGX_OPC_SHLXI, - TILEGX_OPC_SHRS, - TILEGX_OPC_SHRSI, - TILEGX_OPC_SHRU, - TILEGX_OPC_SHRUI, - TILEGX_OPC_SHRUX, - TILEGX_OPC_SHRUXI, - TILEGX_OPC_SHUFFLEBYTES, - TILEGX_OPC_ST, - TILEGX_OPC_ST1, - TILEGX_OPC_ST1_ADD, - TILEGX_OPC_ST2, - TILEGX_OPC_ST2_ADD, - TILEGX_OPC_ST4, - TILEGX_OPC_ST4_ADD, - TILEGX_OPC_ST_ADD, - TILEGX_OPC_STNT, - TILEGX_OPC_STNT1, - TILEGX_OPC_STNT1_ADD, - TILEGX_OPC_STNT2, - TILEGX_OPC_STNT2_ADD, - TILEGX_OPC_STNT4, - TILEGX_OPC_STNT4_ADD, - TILEGX_OPC_STNT_ADD, - TILEGX_OPC_SUB, - TILEGX_OPC_SUBX, - TILEGX_OPC_SUBXSC, - TILEGX_OPC_SWINT0, - TILEGX_OPC_SWINT1, - TILEGX_OPC_SWINT2, - TILEGX_OPC_SWINT3, - TILEGX_OPC_TBLIDXB0, - TILEGX_OPC_TBLIDXB1, - TILEGX_OPC_TBLIDXB2, - TILEGX_OPC_TBLIDXB3, - TILEGX_OPC_V1ADD, - TILEGX_OPC_V1ADDI, - TILEGX_OPC_V1ADDUC, - TILEGX_OPC_V1ADIFFU, - TILEGX_OPC_V1AVGU, - TILEGX_OPC_V1CMPEQ, - TILEGX_OPC_V1CMPEQI, - TILEGX_OPC_V1CMPLES, - TILEGX_OPC_V1CMPLEU, - TILEGX_OPC_V1CMPLTS, - TILEGX_OPC_V1CMPLTSI, - TILEGX_OPC_V1CMPLTU, - TILEGX_OPC_V1CMPLTUI, - TILEGX_OPC_V1CMPNE, - TILEGX_OPC_V1DDOTPU, - TILEGX_OPC_V1DDOTPUA, - TILEGX_OPC_V1DDOTPUS, - TILEGX_OPC_V1DDOTPUSA, - TILEGX_OPC_V1DOTP, - TILEGX_OPC_V1DOTPA, - TILEGX_OPC_V1DOTPU, - TILEGX_OPC_V1DOTPUA, - TILEGX_OPC_V1DOTPUS, - TILEGX_OPC_V1DOTPUSA, - TILEGX_OPC_V1INT_H, - TILEGX_OPC_V1INT_L, - TILEGX_OPC_V1MAXU, - TILEGX_OPC_V1MAXUI, - TILEGX_OPC_V1MINU, - TILEGX_OPC_V1MINUI, - TILEGX_OPC_V1MNZ, - TILEGX_OPC_V1MULTU, - TILEGX_OPC_V1MULU, - TILEGX_OPC_V1MULUS, - TILEGX_OPC_V1MZ, - TILEGX_OPC_V1SADAU, - TILEGX_OPC_V1SADU, - TILEGX_OPC_V1SHL, - TILEGX_OPC_V1SHLI, - TILEGX_OPC_V1SHRS, - TILEGX_OPC_V1SHRSI, - TILEGX_OPC_V1SHRU, - TILEGX_OPC_V1SHRUI, - TILEGX_OPC_V1SUB, - TILEGX_OPC_V1SUBUC, - TILEGX_OPC_V2ADD, - TILEGX_OPC_V2ADDI, - TILEGX_OPC_V2ADDSC, - TILEGX_OPC_V2ADIFFS, - TILEGX_OPC_V2AVGS, - TILEGX_OPC_V2CMPEQ, - TILEGX_OPC_V2CMPEQI, - TILEGX_OPC_V2CMPLES, - TILEGX_OPC_V2CMPLEU, - TILEGX_OPC_V2CMPLTS, - TILEGX_OPC_V2CMPLTSI, - TILEGX_OPC_V2CMPLTU, - TILEGX_OPC_V2CMPLTUI, - TILEGX_OPC_V2CMPNE, - TILEGX_OPC_V2DOTP, - TILEGX_OPC_V2DOTPA, - TILEGX_OPC_V2INT_H, - TILEGX_OPC_V2INT_L, - TILEGX_OPC_V2MAXS, - TILEGX_OPC_V2MAXSI, - TILEGX_OPC_V2MINS, - TILEGX_OPC_V2MINSI, - TILEGX_OPC_V2MNZ, - TILEGX_OPC_V2MULFSC, - TILEGX_OPC_V2MULS, - TILEGX_OPC_V2MULTS, - TILEGX_OPC_V2MZ, - TILEGX_OPC_V2PACKH, - TILEGX_OPC_V2PACKL, - TILEGX_OPC_V2PACKUC, - TILEGX_OPC_V2SADAS, - TILEGX_OPC_V2SADAU, - TILEGX_OPC_V2SADS, - TILEGX_OPC_V2SADU, - TILEGX_OPC_V2SHL, - TILEGX_OPC_V2SHLI, - TILEGX_OPC_V2SHLSC, - TILEGX_OPC_V2SHRS, - TILEGX_OPC_V2SHRSI, - TILEGX_OPC_V2SHRU, - TILEGX_OPC_V2SHRUI, - TILEGX_OPC_V2SUB, - TILEGX_OPC_V2SUBSC, - TILEGX_OPC_V4ADD, - TILEGX_OPC_V4ADDSC, - TILEGX_OPC_V4INT_H, - TILEGX_OPC_V4INT_L, - TILEGX_OPC_V4PACKSC, - TILEGX_OPC_V4SHL, - TILEGX_OPC_V4SHLSC, - TILEGX_OPC_V4SHRS, - TILEGX_OPC_V4SHRU, - TILEGX_OPC_V4SUB, - TILEGX_OPC_V4SUBSC, - TILEGX_OPC_WH64, - TILEGX_OPC_XOR, - TILEGX_OPC_XORI, - TILEGX_OPC_NONE -} tilegx_mnemonic; - -enum -{ - TILEGX_MAX_OPERANDS = 4 /* bfexts */ -}; - -struct tilegx_opcode -{ - /* The opcode mnemonic, e.g. "add" */ - const char *name; - - /* The enum value for this mnemonic. */ - tilegx_mnemonic mnemonic; - - /* A bit mask of which of the five pipes this instruction - is compatible with: - X0 0x01 - X1 0x02 - Y0 0x04 - Y1 0x08 - Y2 0x10 */ - unsigned char pipes; - - /* How many operands are there? */ - unsigned char num_operands; - - /* Which register does this write implicitly, or TREG_ZERO if none? */ - unsigned char implicitly_written_register; - - /* Can this be bundled with other instructions (almost always true). */ - unsigned char can_bundle; - - /* The description of the operands. Each of these is an - * index into the tilegx_operands[] table. */ - unsigned char operands[TILEGX_NUM_PIPELINE_ENCODINGS][TILEGX_MAX_OPERANDS]; - - /* A mask of which bits have predefined values for each pipeline. - * This is useful for disassembly. */ - tilegx_bundle_bits fixed_bit_masks[TILEGX_NUM_PIPELINE_ENCODINGS]; - - /* For each bit set in fixed_bit_masks, what the value is for this - * instruction. */ - tilegx_bundle_bits fixed_bit_values[TILEGX_NUM_PIPELINE_ENCODINGS]; -}; - -/* Used for non-textual disassembly into structs. */ -struct tilegx_decoded_instruction -{ - const struct tilegx_opcode *opcode; - const struct tilegx_operand *operands[TILEGX_MAX_OPERANDS]; - long long operand_values[TILEGX_MAX_OPERANDS]; -}; - -enum -{ - ADDI_IMM8_OPCODE_X0 = 1, - ADDI_IMM8_OPCODE_X1 = 1, - ADDI_OPCODE_Y0 = 0, - ADDI_OPCODE_Y1 = 1, - ADDLI_OPCODE_X0 = 1, - ADDLI_OPCODE_X1 = 0, - ADDXI_IMM8_OPCODE_X0 = 2, - ADDXI_IMM8_OPCODE_X1 = 2, - ADDXI_OPCODE_Y0 = 1, - ADDXI_OPCODE_Y1 = 2, - ADDXLI_OPCODE_X0 = 2, - ADDXLI_OPCODE_X1 = 1, - ADDXSC_RRR_0_OPCODE_X0 = 1, - ADDXSC_RRR_0_OPCODE_X1 = 1, - ADDX_RRR_0_OPCODE_X0 = 2, - ADDX_RRR_0_OPCODE_X1 = 2, - ADDX_RRR_0_OPCODE_Y0 = 0, - ADDX_SPECIAL_0_OPCODE_Y1 = 0, - ADD_RRR_0_OPCODE_X0 = 3, - ADD_RRR_0_OPCODE_X1 = 3, - ADD_RRR_0_OPCODE_Y0 = 1, - ADD_SPECIAL_0_OPCODE_Y1 = 1, - ANDI_IMM8_OPCODE_X0 = 3, - ANDI_IMM8_OPCODE_X1 = 3, - ANDI_OPCODE_Y0 = 2, - ANDI_OPCODE_Y1 = 3, - AND_RRR_0_OPCODE_X0 = 4, - AND_RRR_0_OPCODE_X1 = 4, - AND_RRR_5_OPCODE_Y0 = 0, - AND_RRR_5_OPCODE_Y1 = 0, - BEQZT_BRANCH_OPCODE_X1 = 16, - BEQZ_BRANCH_OPCODE_X1 = 17, - BFEXTS_BF_OPCODE_X0 = 4, - BFEXTU_BF_OPCODE_X0 = 5, - BFINS_BF_OPCODE_X0 = 6, - BF_OPCODE_X0 = 3, - BGEZT_BRANCH_OPCODE_X1 = 18, - BGEZ_BRANCH_OPCODE_X1 = 19, - BGTZT_BRANCH_OPCODE_X1 = 20, - BGTZ_BRANCH_OPCODE_X1 = 21, - BLBCT_BRANCH_OPCODE_X1 = 22, - BLBC_BRANCH_OPCODE_X1 = 23, - BLBST_BRANCH_OPCODE_X1 = 24, - BLBS_BRANCH_OPCODE_X1 = 25, - BLEZT_BRANCH_OPCODE_X1 = 26, - BLEZ_BRANCH_OPCODE_X1 = 27, - BLTZT_BRANCH_OPCODE_X1 = 28, - BLTZ_BRANCH_OPCODE_X1 = 29, - BNEZT_BRANCH_OPCODE_X1 = 30, - BNEZ_BRANCH_OPCODE_X1 = 31, - BRANCH_OPCODE_X1 = 2, - CMOVEQZ_RRR_0_OPCODE_X0 = 5, - CMOVEQZ_RRR_4_OPCODE_Y0 = 0, - CMOVNEZ_RRR_0_OPCODE_X0 = 6, - CMOVNEZ_RRR_4_OPCODE_Y0 = 1, - CMPEQI_IMM8_OPCODE_X0 = 4, - CMPEQI_IMM8_OPCODE_X1 = 4, - CMPEQI_OPCODE_Y0 = 3, - CMPEQI_OPCODE_Y1 = 4, - CMPEQ_RRR_0_OPCODE_X0 = 7, - CMPEQ_RRR_0_OPCODE_X1 = 5, - CMPEQ_RRR_3_OPCODE_Y0 = 0, - CMPEQ_RRR_3_OPCODE_Y1 = 2, - CMPEXCH4_RRR_0_OPCODE_X1 = 6, - CMPEXCH_RRR_0_OPCODE_X1 = 7, - CMPLES_RRR_0_OPCODE_X0 = 8, - CMPLES_RRR_0_OPCODE_X1 = 8, - CMPLES_RRR_2_OPCODE_Y0 = 0, - CMPLES_RRR_2_OPCODE_Y1 = 0, - CMPLEU_RRR_0_OPCODE_X0 = 9, - CMPLEU_RRR_0_OPCODE_X1 = 9, - CMPLEU_RRR_2_OPCODE_Y0 = 1, - CMPLEU_RRR_2_OPCODE_Y1 = 1, - CMPLTSI_IMM8_OPCODE_X0 = 5, - CMPLTSI_IMM8_OPCODE_X1 = 5, - CMPLTSI_OPCODE_Y0 = 4, - CMPLTSI_OPCODE_Y1 = 5, - CMPLTS_RRR_0_OPCODE_X0 = 10, - CMPLTS_RRR_0_OPCODE_X1 = 10, - CMPLTS_RRR_2_OPCODE_Y0 = 2, - CMPLTS_RRR_2_OPCODE_Y1 = 2, - CMPLTUI_IMM8_OPCODE_X0 = 6, - CMPLTUI_IMM8_OPCODE_X1 = 6, - CMPLTU_RRR_0_OPCODE_X0 = 11, - CMPLTU_RRR_0_OPCODE_X1 = 11, - CMPLTU_RRR_2_OPCODE_Y0 = 3, - CMPLTU_RRR_2_OPCODE_Y1 = 3, - CMPNE_RRR_0_OPCODE_X0 = 12, - CMPNE_RRR_0_OPCODE_X1 = 12, - CMPNE_RRR_3_OPCODE_Y0 = 1, - CMPNE_RRR_3_OPCODE_Y1 = 3, - CMULAF_RRR_0_OPCODE_X0 = 13, - CMULA_RRR_0_OPCODE_X0 = 14, - CMULFR_RRR_0_OPCODE_X0 = 15, - CMULF_RRR_0_OPCODE_X0 = 16, - CMULHR_RRR_0_OPCODE_X0 = 17, - CMULH_RRR_0_OPCODE_X0 = 18, - CMUL_RRR_0_OPCODE_X0 = 19, - CNTLZ_UNARY_OPCODE_X0 = 1, - CNTLZ_UNARY_OPCODE_Y0 = 1, - CNTTZ_UNARY_OPCODE_X0 = 2, - CNTTZ_UNARY_OPCODE_Y0 = 2, - CRC32_32_RRR_0_OPCODE_X0 = 20, - CRC32_8_RRR_0_OPCODE_X0 = 21, - DBLALIGN2_RRR_0_OPCODE_X0 = 22, - DBLALIGN2_RRR_0_OPCODE_X1 = 13, - DBLALIGN4_RRR_0_OPCODE_X0 = 23, - DBLALIGN4_RRR_0_OPCODE_X1 = 14, - DBLALIGN6_RRR_0_OPCODE_X0 = 24, - DBLALIGN6_RRR_0_OPCODE_X1 = 15, - DBLALIGN_RRR_0_OPCODE_X0 = 25, - DRAIN_UNARY_OPCODE_X1 = 1, - DTLBPR_UNARY_OPCODE_X1 = 2, - EXCH4_RRR_0_OPCODE_X1 = 16, - EXCH_RRR_0_OPCODE_X1 = 17, - FDOUBLE_ADDSUB_RRR_0_OPCODE_X0 = 26, - FDOUBLE_ADD_FLAGS_RRR_0_OPCODE_X0 = 27, - FDOUBLE_MUL_FLAGS_RRR_0_OPCODE_X0 = 28, - FDOUBLE_PACK1_RRR_0_OPCODE_X0 = 29, - FDOUBLE_PACK2_RRR_0_OPCODE_X0 = 30, - FDOUBLE_SUB_FLAGS_RRR_0_OPCODE_X0 = 31, - FDOUBLE_UNPACK_MAX_RRR_0_OPCODE_X0 = 32, - FDOUBLE_UNPACK_MIN_RRR_0_OPCODE_X0 = 33, - FETCHADD4_RRR_0_OPCODE_X1 = 18, - FETCHADDGEZ4_RRR_0_OPCODE_X1 = 19, - FETCHADDGEZ_RRR_0_OPCODE_X1 = 20, - FETCHADD_RRR_0_OPCODE_X1 = 21, - FETCHAND4_RRR_0_OPCODE_X1 = 22, - FETCHAND_RRR_0_OPCODE_X1 = 23, - FETCHOR4_RRR_0_OPCODE_X1 = 24, - FETCHOR_RRR_0_OPCODE_X1 = 25, - FINV_UNARY_OPCODE_X1 = 3, - FLUSHWB_UNARY_OPCODE_X1 = 4, - FLUSH_UNARY_OPCODE_X1 = 5, - FNOP_UNARY_OPCODE_X0 = 3, - FNOP_UNARY_OPCODE_X1 = 6, - FNOP_UNARY_OPCODE_Y0 = 3, - FNOP_UNARY_OPCODE_Y1 = 8, - FSINGLE_ADD1_RRR_0_OPCODE_X0 = 34, - FSINGLE_ADDSUB2_RRR_0_OPCODE_X0 = 35, - FSINGLE_MUL1_RRR_0_OPCODE_X0 = 36, - FSINGLE_MUL2_RRR_0_OPCODE_X0 = 37, - FSINGLE_PACK1_UNARY_OPCODE_X0 = 4, - FSINGLE_PACK1_UNARY_OPCODE_Y0 = 4, - FSINGLE_PACK2_RRR_0_OPCODE_X0 = 38, - FSINGLE_SUB1_RRR_0_OPCODE_X0 = 39, - ICOH_UNARY_OPCODE_X1 = 7, - ILL_UNARY_OPCODE_X1 = 8, - ILL_UNARY_OPCODE_Y1 = 9, - IMM8_OPCODE_X0 = 4, - IMM8_OPCODE_X1 = 3, - INV_UNARY_OPCODE_X1 = 9, - IRET_UNARY_OPCODE_X1 = 10, - JALRP_UNARY_OPCODE_X1 = 11, - JALRP_UNARY_OPCODE_Y1 = 10, - JALR_UNARY_OPCODE_X1 = 12, - JALR_UNARY_OPCODE_Y1 = 11, - JAL_JUMP_OPCODE_X1 = 0, - JRP_UNARY_OPCODE_X1 = 13, - JRP_UNARY_OPCODE_Y1 = 12, - JR_UNARY_OPCODE_X1 = 14, - JR_UNARY_OPCODE_Y1 = 13, - JUMP_OPCODE_X1 = 4, - J_JUMP_OPCODE_X1 = 1, - LD1S_ADD_IMM8_OPCODE_X1 = 7, - LD1S_OPCODE_Y2 = 0, - LD1S_UNARY_OPCODE_X1 = 15, - LD1U_ADD_IMM8_OPCODE_X1 = 8, - LD1U_OPCODE_Y2 = 1, - LD1U_UNARY_OPCODE_X1 = 16, - LD2S_ADD_IMM8_OPCODE_X1 = 9, - LD2S_OPCODE_Y2 = 2, - LD2S_UNARY_OPCODE_X1 = 17, - LD2U_ADD_IMM8_OPCODE_X1 = 10, - LD2U_OPCODE_Y2 = 3, - LD2U_UNARY_OPCODE_X1 = 18, - LD4S_ADD_IMM8_OPCODE_X1 = 11, - LD4S_OPCODE_Y2 = 1, - LD4S_UNARY_OPCODE_X1 = 19, - LD4U_ADD_IMM8_OPCODE_X1 = 12, - LD4U_OPCODE_Y2 = 2, - LD4U_UNARY_OPCODE_X1 = 20, - LDNA_UNARY_OPCODE_X1 = 21, - LDNT1S_ADD_IMM8_OPCODE_X1 = 13, - LDNT1S_UNARY_OPCODE_X1 = 22, - LDNT1U_ADD_IMM8_OPCODE_X1 = 14, - LDNT1U_UNARY_OPCODE_X1 = 23, - LDNT2S_ADD_IMM8_OPCODE_X1 = 15, - LDNT2S_UNARY_OPCODE_X1 = 24, - LDNT2U_ADD_IMM8_OPCODE_X1 = 16, - LDNT2U_UNARY_OPCODE_X1 = 25, - LDNT4S_ADD_IMM8_OPCODE_X1 = 17, - LDNT4S_UNARY_OPCODE_X1 = 26, - LDNT4U_ADD_IMM8_OPCODE_X1 = 18, - LDNT4U_UNARY_OPCODE_X1 = 27, - LDNT_ADD_IMM8_OPCODE_X1 = 19, - LDNT_UNARY_OPCODE_X1 = 28, - LD_ADD_IMM8_OPCODE_X1 = 20, - LD_OPCODE_Y2 = 3, - LD_UNARY_OPCODE_X1 = 29, - LNK_UNARY_OPCODE_X1 = 30, - LNK_UNARY_OPCODE_Y1 = 14, - LWNA_ADD_IMM8_OPCODE_X1 = 21, - MFSPR_IMM8_OPCODE_X1 = 22, - MF_UNARY_OPCODE_X1 = 31, - MM_BF_OPCODE_X0 = 7, - MNZ_RRR_0_OPCODE_X0 = 40, - MNZ_RRR_0_OPCODE_X1 = 26, - MNZ_RRR_4_OPCODE_Y0 = 2, - MNZ_RRR_4_OPCODE_Y1 = 2, - MODE_OPCODE_YA2 = 1, - MODE_OPCODE_YB2 = 2, - MODE_OPCODE_YC2 = 3, - MTSPR_IMM8_OPCODE_X1 = 23, - MULAX_RRR_0_OPCODE_X0 = 41, - MULAX_RRR_3_OPCODE_Y0 = 2, - MULA_HS_HS_RRR_0_OPCODE_X0 = 42, - MULA_HS_HS_RRR_9_OPCODE_Y0 = 0, - MULA_HS_HU_RRR_0_OPCODE_X0 = 43, - MULA_HS_LS_RRR_0_OPCODE_X0 = 44, - MULA_HS_LU_RRR_0_OPCODE_X0 = 45, - MULA_HU_HU_RRR_0_OPCODE_X0 = 46, - MULA_HU_HU_RRR_9_OPCODE_Y0 = 1, - MULA_HU_LS_RRR_0_OPCODE_X0 = 47, - MULA_HU_LU_RRR_0_OPCODE_X0 = 48, - MULA_LS_LS_RRR_0_OPCODE_X0 = 49, - MULA_LS_LS_RRR_9_OPCODE_Y0 = 2, - MULA_LS_LU_RRR_0_OPCODE_X0 = 50, - MULA_LU_LU_RRR_0_OPCODE_X0 = 51, - MULA_LU_LU_RRR_9_OPCODE_Y0 = 3, - MULX_RRR_0_OPCODE_X0 = 52, - MULX_RRR_3_OPCODE_Y0 = 3, - MUL_HS_HS_RRR_0_OPCODE_X0 = 53, - MUL_HS_HS_RRR_8_OPCODE_Y0 = 0, - MUL_HS_HU_RRR_0_OPCODE_X0 = 54, - MUL_HS_LS_RRR_0_OPCODE_X0 = 55, - MUL_HS_LU_RRR_0_OPCODE_X0 = 56, - MUL_HU_HU_RRR_0_OPCODE_X0 = 57, - MUL_HU_HU_RRR_8_OPCODE_Y0 = 1, - MUL_HU_LS_RRR_0_OPCODE_X0 = 58, - MUL_HU_LU_RRR_0_OPCODE_X0 = 59, - MUL_LS_LS_RRR_0_OPCODE_X0 = 60, - MUL_LS_LS_RRR_8_OPCODE_Y0 = 2, - MUL_LS_LU_RRR_0_OPCODE_X0 = 61, - MUL_LU_LU_RRR_0_OPCODE_X0 = 62, - MUL_LU_LU_RRR_8_OPCODE_Y0 = 3, - MZ_RRR_0_OPCODE_X0 = 63, - MZ_RRR_0_OPCODE_X1 = 27, - MZ_RRR_4_OPCODE_Y0 = 3, - MZ_RRR_4_OPCODE_Y1 = 3, - NAP_UNARY_OPCODE_X1 = 32, - NOP_UNARY_OPCODE_X0 = 5, - NOP_UNARY_OPCODE_X1 = 33, - NOP_UNARY_OPCODE_Y0 = 5, - NOP_UNARY_OPCODE_Y1 = 15, - NOR_RRR_0_OPCODE_X0 = 64, - NOR_RRR_0_OPCODE_X1 = 28, - NOR_RRR_5_OPCODE_Y0 = 1, - NOR_RRR_5_OPCODE_Y1 = 1, - ORI_IMM8_OPCODE_X0 = 7, - ORI_IMM8_OPCODE_X1 = 24, - OR_RRR_0_OPCODE_X0 = 65, - OR_RRR_0_OPCODE_X1 = 29, - OR_RRR_5_OPCODE_Y0 = 2, - OR_RRR_5_OPCODE_Y1 = 2, - PCNT_UNARY_OPCODE_X0 = 6, - PCNT_UNARY_OPCODE_Y0 = 6, - REVBITS_UNARY_OPCODE_X0 = 7, - REVBITS_UNARY_OPCODE_Y0 = 7, - REVBYTES_UNARY_OPCODE_X0 = 8, - REVBYTES_UNARY_OPCODE_Y0 = 8, - ROTLI_SHIFT_OPCODE_X0 = 1, - ROTLI_SHIFT_OPCODE_X1 = 1, - ROTLI_SHIFT_OPCODE_Y0 = 0, - ROTLI_SHIFT_OPCODE_Y1 = 0, - ROTL_RRR_0_OPCODE_X0 = 66, - ROTL_RRR_0_OPCODE_X1 = 30, - ROTL_RRR_6_OPCODE_Y0 = 0, - ROTL_RRR_6_OPCODE_Y1 = 0, - RRR_0_OPCODE_X0 = 5, - RRR_0_OPCODE_X1 = 5, - RRR_0_OPCODE_Y0 = 5, - RRR_0_OPCODE_Y1 = 6, - RRR_1_OPCODE_Y0 = 6, - RRR_1_OPCODE_Y1 = 7, - RRR_2_OPCODE_Y0 = 7, - RRR_2_OPCODE_Y1 = 8, - RRR_3_OPCODE_Y0 = 8, - RRR_3_OPCODE_Y1 = 9, - RRR_4_OPCODE_Y0 = 9, - RRR_4_OPCODE_Y1 = 10, - RRR_5_OPCODE_Y0 = 10, - RRR_5_OPCODE_Y1 = 11, - RRR_6_OPCODE_Y0 = 11, - RRR_6_OPCODE_Y1 = 12, - RRR_7_OPCODE_Y0 = 12, - RRR_7_OPCODE_Y1 = 13, - RRR_8_OPCODE_Y0 = 13, - RRR_9_OPCODE_Y0 = 14, - SHIFT_OPCODE_X0 = 6, - SHIFT_OPCODE_X1 = 6, - SHIFT_OPCODE_Y0 = 15, - SHIFT_OPCODE_Y1 = 14, - SHL16INSLI_OPCODE_X0 = 7, - SHL16INSLI_OPCODE_X1 = 7, - SHL1ADDX_RRR_0_OPCODE_X0 = 67, - SHL1ADDX_RRR_0_OPCODE_X1 = 31, - SHL1ADDX_RRR_7_OPCODE_Y0 = 1, - SHL1ADDX_RRR_7_OPCODE_Y1 = 1, - SHL1ADD_RRR_0_OPCODE_X0 = 68, - SHL1ADD_RRR_0_OPCODE_X1 = 32, - SHL1ADD_RRR_1_OPCODE_Y0 = 0, - SHL1ADD_RRR_1_OPCODE_Y1 = 0, - SHL2ADDX_RRR_0_OPCODE_X0 = 69, - SHL2ADDX_RRR_0_OPCODE_X1 = 33, - SHL2ADDX_RRR_7_OPCODE_Y0 = 2, - SHL2ADDX_RRR_7_OPCODE_Y1 = 2, - SHL2ADD_RRR_0_OPCODE_X0 = 70, - SHL2ADD_RRR_0_OPCODE_X1 = 34, - SHL2ADD_RRR_1_OPCODE_Y0 = 1, - SHL2ADD_RRR_1_OPCODE_Y1 = 1, - SHL3ADDX_RRR_0_OPCODE_X0 = 71, - SHL3ADDX_RRR_0_OPCODE_X1 = 35, - SHL3ADDX_RRR_7_OPCODE_Y0 = 3, - SHL3ADDX_RRR_7_OPCODE_Y1 = 3, - SHL3ADD_RRR_0_OPCODE_X0 = 72, - SHL3ADD_RRR_0_OPCODE_X1 = 36, - SHL3ADD_RRR_1_OPCODE_Y0 = 2, - SHL3ADD_RRR_1_OPCODE_Y1 = 2, - SHLI_SHIFT_OPCODE_X0 = 2, - SHLI_SHIFT_OPCODE_X1 = 2, - SHLI_SHIFT_OPCODE_Y0 = 1, - SHLI_SHIFT_OPCODE_Y1 = 1, - SHLXI_SHIFT_OPCODE_X0 = 3, - SHLXI_SHIFT_OPCODE_X1 = 3, - SHLX_RRR_0_OPCODE_X0 = 73, - SHLX_RRR_0_OPCODE_X1 = 37, - SHL_RRR_0_OPCODE_X0 = 74, - SHL_RRR_0_OPCODE_X1 = 38, - SHL_RRR_6_OPCODE_Y0 = 1, - SHL_RRR_6_OPCODE_Y1 = 1, - SHRSI_SHIFT_OPCODE_X0 = 4, - SHRSI_SHIFT_OPCODE_X1 = 4, - SHRSI_SHIFT_OPCODE_Y0 = 2, - SHRSI_SHIFT_OPCODE_Y1 = 2, - SHRS_RRR_0_OPCODE_X0 = 75, - SHRS_RRR_0_OPCODE_X1 = 39, - SHRS_RRR_6_OPCODE_Y0 = 2, - SHRS_RRR_6_OPCODE_Y1 = 2, - SHRUI_SHIFT_OPCODE_X0 = 5, - SHRUI_SHIFT_OPCODE_X1 = 5, - SHRUI_SHIFT_OPCODE_Y0 = 3, - SHRUI_SHIFT_OPCODE_Y1 = 3, - SHRUXI_SHIFT_OPCODE_X0 = 6, - SHRUXI_SHIFT_OPCODE_X1 = 6, - SHRUX_RRR_0_OPCODE_X0 = 76, - SHRUX_RRR_0_OPCODE_X1 = 40, - SHRU_RRR_0_OPCODE_X0 = 77, - SHRU_RRR_0_OPCODE_X1 = 41, - SHRU_RRR_6_OPCODE_Y0 = 3, - SHRU_RRR_6_OPCODE_Y1 = 3, - SHUFFLEBYTES_RRR_0_OPCODE_X0 = 78, - ST1_ADD_IMM8_OPCODE_X1 = 25, - ST1_OPCODE_Y2 = 0, - ST1_RRR_0_OPCODE_X1 = 42, - ST2_ADD_IMM8_OPCODE_X1 = 26, - ST2_OPCODE_Y2 = 1, - ST2_RRR_0_OPCODE_X1 = 43, - ST4_ADD_IMM8_OPCODE_X1 = 27, - ST4_OPCODE_Y2 = 2, - ST4_RRR_0_OPCODE_X1 = 44, - STNT1_ADD_IMM8_OPCODE_X1 = 28, - STNT1_RRR_0_OPCODE_X1 = 45, - STNT2_ADD_IMM8_OPCODE_X1 = 29, - STNT2_RRR_0_OPCODE_X1 = 46, - STNT4_ADD_IMM8_OPCODE_X1 = 30, - STNT4_RRR_0_OPCODE_X1 = 47, - STNT_ADD_IMM8_OPCODE_X1 = 31, - STNT_RRR_0_OPCODE_X1 = 48, - ST_ADD_IMM8_OPCODE_X1 = 32, - ST_OPCODE_Y2 = 3, - ST_RRR_0_OPCODE_X1 = 49, - SUBXSC_RRR_0_OPCODE_X0 = 79, - SUBXSC_RRR_0_OPCODE_X1 = 50, - SUBX_RRR_0_OPCODE_X0 = 80, - SUBX_RRR_0_OPCODE_X1 = 51, - SUBX_RRR_0_OPCODE_Y0 = 2, - SUBX_RRR_0_OPCODE_Y1 = 2, - SUB_RRR_0_OPCODE_X0 = 81, - SUB_RRR_0_OPCODE_X1 = 52, - SUB_RRR_0_OPCODE_Y0 = 3, - SUB_RRR_0_OPCODE_Y1 = 3, - SWINT0_UNARY_OPCODE_X1 = 34, - SWINT1_UNARY_OPCODE_X1 = 35, - SWINT2_UNARY_OPCODE_X1 = 36, - SWINT3_UNARY_OPCODE_X1 = 37, - TBLIDXB0_UNARY_OPCODE_X0 = 9, - TBLIDXB0_UNARY_OPCODE_Y0 = 9, - TBLIDXB1_UNARY_OPCODE_X0 = 10, - TBLIDXB1_UNARY_OPCODE_Y0 = 10, - TBLIDXB2_UNARY_OPCODE_X0 = 11, - TBLIDXB2_UNARY_OPCODE_Y0 = 11, - TBLIDXB3_UNARY_OPCODE_X0 = 12, - TBLIDXB3_UNARY_OPCODE_Y0 = 12, - UNARY_RRR_0_OPCODE_X0 = 82, - UNARY_RRR_0_OPCODE_X1 = 53, - UNARY_RRR_1_OPCODE_Y0 = 3, - UNARY_RRR_1_OPCODE_Y1 = 3, - V1ADDI_IMM8_OPCODE_X0 = 8, - V1ADDI_IMM8_OPCODE_X1 = 33, - V1ADDUC_RRR_0_OPCODE_X0 = 83, - V1ADDUC_RRR_0_OPCODE_X1 = 54, - V1ADD_RRR_0_OPCODE_X0 = 84, - V1ADD_RRR_0_OPCODE_X1 = 55, - V1ADIFFU_RRR_0_OPCODE_X0 = 85, - V1AVGU_RRR_0_OPCODE_X0 = 86, - V1CMPEQI_IMM8_OPCODE_X0 = 9, - V1CMPEQI_IMM8_OPCODE_X1 = 34, - V1CMPEQ_RRR_0_OPCODE_X0 = 87, - V1CMPEQ_RRR_0_OPCODE_X1 = 56, - V1CMPLES_RRR_0_OPCODE_X0 = 88, - V1CMPLES_RRR_0_OPCODE_X1 = 57, - V1CMPLEU_RRR_0_OPCODE_X0 = 89, - V1CMPLEU_RRR_0_OPCODE_X1 = 58, - V1CMPLTSI_IMM8_OPCODE_X0 = 10, - V1CMPLTSI_IMM8_OPCODE_X1 = 35, - V1CMPLTS_RRR_0_OPCODE_X0 = 90, - V1CMPLTS_RRR_0_OPCODE_X1 = 59, - V1CMPLTUI_IMM8_OPCODE_X0 = 11, - V1CMPLTUI_IMM8_OPCODE_X1 = 36, - V1CMPLTU_RRR_0_OPCODE_X0 = 91, - V1CMPLTU_RRR_0_OPCODE_X1 = 60, - V1CMPNE_RRR_0_OPCODE_X0 = 92, - V1CMPNE_RRR_0_OPCODE_X1 = 61, - V1DDOTPUA_RRR_0_OPCODE_X0 = 161, - V1DDOTPUSA_RRR_0_OPCODE_X0 = 93, - V1DDOTPUS_RRR_0_OPCODE_X0 = 94, - V1DDOTPU_RRR_0_OPCODE_X0 = 162, - V1DOTPA_RRR_0_OPCODE_X0 = 95, - V1DOTPUA_RRR_0_OPCODE_X0 = 163, - V1DOTPUSA_RRR_0_OPCODE_X0 = 96, - V1DOTPUS_RRR_0_OPCODE_X0 = 97, - V1DOTPU_RRR_0_OPCODE_X0 = 164, - V1DOTP_RRR_0_OPCODE_X0 = 98, - V1INT_H_RRR_0_OPCODE_X0 = 99, - V1INT_H_RRR_0_OPCODE_X1 = 62, - V1INT_L_RRR_0_OPCODE_X0 = 100, - V1INT_L_RRR_0_OPCODE_X1 = 63, - V1MAXUI_IMM8_OPCODE_X0 = 12, - V1MAXUI_IMM8_OPCODE_X1 = 37, - V1MAXU_RRR_0_OPCODE_X0 = 101, - V1MAXU_RRR_0_OPCODE_X1 = 64, - V1MINUI_IMM8_OPCODE_X0 = 13, - V1MINUI_IMM8_OPCODE_X1 = 38, - V1MINU_RRR_0_OPCODE_X0 = 102, - V1MINU_RRR_0_OPCODE_X1 = 65, - V1MNZ_RRR_0_OPCODE_X0 = 103, - V1MNZ_RRR_0_OPCODE_X1 = 66, - V1MULTU_RRR_0_OPCODE_X0 = 104, - V1MULUS_RRR_0_OPCODE_X0 = 105, - V1MULU_RRR_0_OPCODE_X0 = 106, - V1MZ_RRR_0_OPCODE_X0 = 107, - V1MZ_RRR_0_OPCODE_X1 = 67, - V1SADAU_RRR_0_OPCODE_X0 = 108, - V1SADU_RRR_0_OPCODE_X0 = 109, - V1SHLI_SHIFT_OPCODE_X0 = 7, - V1SHLI_SHIFT_OPCODE_X1 = 7, - V1SHL_RRR_0_OPCODE_X0 = 110, - V1SHL_RRR_0_OPCODE_X1 = 68, - V1SHRSI_SHIFT_OPCODE_X0 = 8, - V1SHRSI_SHIFT_OPCODE_X1 = 8, - V1SHRS_RRR_0_OPCODE_X0 = 111, - V1SHRS_RRR_0_OPCODE_X1 = 69, - V1SHRUI_SHIFT_OPCODE_X0 = 9, - V1SHRUI_SHIFT_OPCODE_X1 = 9, - V1SHRU_RRR_0_OPCODE_X0 = 112, - V1SHRU_RRR_0_OPCODE_X1 = 70, - V1SUBUC_RRR_0_OPCODE_X0 = 113, - V1SUBUC_RRR_0_OPCODE_X1 = 71, - V1SUB_RRR_0_OPCODE_X0 = 114, - V1SUB_RRR_0_OPCODE_X1 = 72, - V2ADDI_IMM8_OPCODE_X0 = 14, - V2ADDI_IMM8_OPCODE_X1 = 39, - V2ADDSC_RRR_0_OPCODE_X0 = 115, - V2ADDSC_RRR_0_OPCODE_X1 = 73, - V2ADD_RRR_0_OPCODE_X0 = 116, - V2ADD_RRR_0_OPCODE_X1 = 74, - V2ADIFFS_RRR_0_OPCODE_X0 = 117, - V2AVGS_RRR_0_OPCODE_X0 = 118, - V2CMPEQI_IMM8_OPCODE_X0 = 15, - V2CMPEQI_IMM8_OPCODE_X1 = 40, - V2CMPEQ_RRR_0_OPCODE_X0 = 119, - V2CMPEQ_RRR_0_OPCODE_X1 = 75, - V2CMPLES_RRR_0_OPCODE_X0 = 120, - V2CMPLES_RRR_0_OPCODE_X1 = 76, - V2CMPLEU_RRR_0_OPCODE_X0 = 121, - V2CMPLEU_RRR_0_OPCODE_X1 = 77, - V2CMPLTSI_IMM8_OPCODE_X0 = 16, - V2CMPLTSI_IMM8_OPCODE_X1 = 41, - V2CMPLTS_RRR_0_OPCODE_X0 = 122, - V2CMPLTS_RRR_0_OPCODE_X1 = 78, - V2CMPLTUI_IMM8_OPCODE_X0 = 17, - V2CMPLTUI_IMM8_OPCODE_X1 = 42, - V2CMPLTU_RRR_0_OPCODE_X0 = 123, - V2CMPLTU_RRR_0_OPCODE_X1 = 79, - V2CMPNE_RRR_0_OPCODE_X0 = 124, - V2CMPNE_RRR_0_OPCODE_X1 = 80, - V2DOTPA_RRR_0_OPCODE_X0 = 125, - V2DOTP_RRR_0_OPCODE_X0 = 126, - V2INT_H_RRR_0_OPCODE_X0 = 127, - V2INT_H_RRR_0_OPCODE_X1 = 81, - V2INT_L_RRR_0_OPCODE_X0 = 128, - V2INT_L_RRR_0_OPCODE_X1 = 82, - V2MAXSI_IMM8_OPCODE_X0 = 18, - V2MAXSI_IMM8_OPCODE_X1 = 43, - V2MAXS_RRR_0_OPCODE_X0 = 129, - V2MAXS_RRR_0_OPCODE_X1 = 83, - V2MINSI_IMM8_OPCODE_X0 = 19, - V2MINSI_IMM8_OPCODE_X1 = 44, - V2MINS_RRR_0_OPCODE_X0 = 130, - V2MINS_RRR_0_OPCODE_X1 = 84, - V2MNZ_RRR_0_OPCODE_X0 = 131, - V2MNZ_RRR_0_OPCODE_X1 = 85, - V2MULFSC_RRR_0_OPCODE_X0 = 132, - V2MULS_RRR_0_OPCODE_X0 = 133, - V2MULTS_RRR_0_OPCODE_X0 = 134, - V2MZ_RRR_0_OPCODE_X0 = 135, - V2MZ_RRR_0_OPCODE_X1 = 86, - V2PACKH_RRR_0_OPCODE_X0 = 136, - V2PACKH_RRR_0_OPCODE_X1 = 87, - V2PACKL_RRR_0_OPCODE_X0 = 137, - V2PACKL_RRR_0_OPCODE_X1 = 88, - V2PACKUC_RRR_0_OPCODE_X0 = 138, - V2PACKUC_RRR_0_OPCODE_X1 = 89, - V2SADAS_RRR_0_OPCODE_X0 = 139, - V2SADAU_RRR_0_OPCODE_X0 = 140, - V2SADS_RRR_0_OPCODE_X0 = 141, - V2SADU_RRR_0_OPCODE_X0 = 142, - V2SHLI_SHIFT_OPCODE_X0 = 10, - V2SHLI_SHIFT_OPCODE_X1 = 10, - V2SHLSC_RRR_0_OPCODE_X0 = 143, - V2SHLSC_RRR_0_OPCODE_X1 = 90, - V2SHL_RRR_0_OPCODE_X0 = 144, - V2SHL_RRR_0_OPCODE_X1 = 91, - V2SHRSI_SHIFT_OPCODE_X0 = 11, - V2SHRSI_SHIFT_OPCODE_X1 = 11, - V2SHRS_RRR_0_OPCODE_X0 = 145, - V2SHRS_RRR_0_OPCODE_X1 = 92, - V2SHRUI_SHIFT_OPCODE_X0 = 12, - V2SHRUI_SHIFT_OPCODE_X1 = 12, - V2SHRU_RRR_0_OPCODE_X0 = 146, - V2SHRU_RRR_0_OPCODE_X1 = 93, - V2SUBSC_RRR_0_OPCODE_X0 = 147, - V2SUBSC_RRR_0_OPCODE_X1 = 94, - V2SUB_RRR_0_OPCODE_X0 = 148, - V2SUB_RRR_0_OPCODE_X1 = 95, - V4ADDSC_RRR_0_OPCODE_X0 = 149, - V4ADDSC_RRR_0_OPCODE_X1 = 96, - V4ADD_RRR_0_OPCODE_X0 = 150, - V4ADD_RRR_0_OPCODE_X1 = 97, - V4INT_H_RRR_0_OPCODE_X0 = 151, - V4INT_H_RRR_0_OPCODE_X1 = 98, - V4INT_L_RRR_0_OPCODE_X0 = 152, - V4INT_L_RRR_0_OPCODE_X1 = 99, - V4PACKSC_RRR_0_OPCODE_X0 = 153, - V4PACKSC_RRR_0_OPCODE_X1 = 100, - V4SHLSC_RRR_0_OPCODE_X0 = 154, - V4SHLSC_RRR_0_OPCODE_X1 = 101, - V4SHL_RRR_0_OPCODE_X0 = 155, - V4SHL_RRR_0_OPCODE_X1 = 102, - V4SHRS_RRR_0_OPCODE_X0 = 156, - V4SHRS_RRR_0_OPCODE_X1 = 103, - V4SHRU_RRR_0_OPCODE_X0 = 157, - V4SHRU_RRR_0_OPCODE_X1 = 104, - V4SUBSC_RRR_0_OPCODE_X0 = 158, - V4SUBSC_RRR_0_OPCODE_X1 = 105, - V4SUB_RRR_0_OPCODE_X0 = 159, - V4SUB_RRR_0_OPCODE_X1 = 106, - WH64_UNARY_OPCODE_X1 = 38, - XORI_IMM8_OPCODE_X0 = 20, - XORI_IMM8_OPCODE_X1 = 45, - XOR_RRR_0_OPCODE_X0 = 160, - XOR_RRR_0_OPCODE_X1 = 107, - XOR_RRR_5_OPCODE_Y0 = 3, - XOR_RRR_5_OPCODE_Y1 = 3 -}; - -static __inline unsigned int -get_BFEnd_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_BFOpcodeExtension_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 24)) & 0xf); -} - -static __inline unsigned int -get_BFStart_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 18)) & 0x3f); -} - -static __inline unsigned int -get_BrOff_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x0000003f) | - (((unsigned int)(n >> 37)) & 0x0001ffc0); -} - -static __inline unsigned int -get_BrType_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 54)) & 0x1f); -} - -static __inline unsigned int -get_Dest_Imm8_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x0000003f) | - (((unsigned int)(n >> 43)) & 0x000000c0); -} - -static __inline unsigned int -get_Dest_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 0)) & 0x3f); -} - -static __inline unsigned int -get_Dest_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x3f); -} - -static __inline unsigned int -get_Dest_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 0)) & 0x3f); -} - -static __inline unsigned int -get_Dest_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x3f); -} - -static __inline unsigned int -get_Imm16_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0xffff); -} - -static __inline unsigned int -get_Imm16_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0xffff); -} - -static __inline unsigned int -get_Imm8OpcodeExtension_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 20)) & 0xff); -} - -static __inline unsigned int -get_Imm8OpcodeExtension_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 51)) & 0xff); -} - -static __inline unsigned int -get_Imm8_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0xff); -} - -static __inline unsigned int -get_Imm8_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0xff); -} - -static __inline unsigned int -get_Imm8_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0xff); -} - -static __inline unsigned int -get_Imm8_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0xff); -} - -static __inline unsigned int -get_JumpOff_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x7ffffff); -} - -static __inline unsigned int -get_JumpOpcodeExtension_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 58)) & 0x1); -} - -static __inline unsigned int -get_MF_Imm14_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 37)) & 0x3fff); -} - -static __inline unsigned int -get_MT_Imm14_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 31)) & 0x0000003f) | - (((unsigned int)(n >> 37)) & 0x00003fc0); -} - -static __inline unsigned int -get_Mode(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 62)) & 0x3); -} - -static __inline unsigned int -get_Opcode_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 28)) & 0x7); -} - -static __inline unsigned int -get_Opcode_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 59)) & 0x7); -} - -static __inline unsigned int -get_Opcode_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 27)) & 0xf); -} - -static __inline unsigned int -get_Opcode_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 58)) & 0xf); -} - -static __inline unsigned int -get_Opcode_Y2(tilegx_bundle_bits n) -{ - return (((n >> 26)) & 0x00000001) | - (((unsigned int)(n >> 56)) & 0x00000002); -} - -static __inline unsigned int -get_RRROpcodeExtension_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 18)) & 0x3ff); -} - -static __inline unsigned int -get_RRROpcodeExtension_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 49)) & 0x3ff); -} - -static __inline unsigned int -get_RRROpcodeExtension_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 18)) & 0x3); -} - -static __inline unsigned int -get_RRROpcodeExtension_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 49)) & 0x3); -} - -static __inline unsigned int -get_ShAmt_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_ShAmt_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline unsigned int -get_ShAmt_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_ShAmt_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline unsigned int -get_ShiftOpcodeExtension_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 18)) & 0x3ff); -} - -static __inline unsigned int -get_ShiftOpcodeExtension_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 49)) & 0x3ff); -} - -static __inline unsigned int -get_ShiftOpcodeExtension_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 18)) & 0x3); -} - -static __inline unsigned int -get_ShiftOpcodeExtension_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 49)) & 0x3); -} - -static __inline unsigned int -get_SrcA_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 6)) & 0x3f); -} - -static __inline unsigned int -get_SrcA_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 37)) & 0x3f); -} - -static __inline unsigned int -get_SrcA_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 6)) & 0x3f); -} - -static __inline unsigned int -get_SrcA_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 37)) & 0x3f); -} - -static __inline unsigned int -get_SrcA_Y2(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 20)) & 0x3f); -} - -static __inline unsigned int -get_SrcBDest_Y2(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 51)) & 0x3f); -} - -static __inline unsigned int -get_SrcB_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_SrcB_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline unsigned int -get_SrcB_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_SrcB_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline unsigned int -get_UnaryOpcodeExtension_X0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_UnaryOpcodeExtension_X1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline unsigned int -get_UnaryOpcodeExtension_Y0(tilegx_bundle_bits num) -{ - const unsigned int n = (unsigned int)num; - return (((n >> 12)) & 0x3f); -} - -static __inline unsigned int -get_UnaryOpcodeExtension_Y1(tilegx_bundle_bits n) -{ - return (((unsigned int)(n >> 43)) & 0x3f); -} - -static __inline int -sign_extend(int n, int num_bits) -{ - int shift = (int)(sizeof(int) * 8 - num_bits); - return (n << shift) >> shift; -} - -static __inline tilegx_bundle_bits -create_BFEnd_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_BFOpcodeExtension_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xf) << 24); -} - -static __inline tilegx_bundle_bits -create_BFStart_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 18); -} - -static __inline tilegx_bundle_bits -create_BrOff_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) | - (((tilegx_bundle_bits)(n & 0x0001ffc0)) << 37); -} - -static __inline tilegx_bundle_bits -create_BrType_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x1f)) << 54); -} - -static __inline tilegx_bundle_bits -create_Dest_Imm8_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) | - (((tilegx_bundle_bits)(n & 0x000000c0)) << 43); -} - -static __inline tilegx_bundle_bits -create_Dest_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 0); -} - -static __inline tilegx_bundle_bits -create_Dest_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 31); -} - -static __inline tilegx_bundle_bits -create_Dest_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 0); -} - -static __inline tilegx_bundle_bits -create_Dest_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 31); -} - -static __inline tilegx_bundle_bits -create_Imm16_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xffff) << 12); -} - -static __inline tilegx_bundle_bits -create_Imm16_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0xffff)) << 43); -} - -static __inline tilegx_bundle_bits -create_Imm8OpcodeExtension_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xff) << 20); -} - -static __inline tilegx_bundle_bits -create_Imm8OpcodeExtension_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0xff)) << 51); -} - -static __inline tilegx_bundle_bits -create_Imm8_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xff) << 12); -} - -static __inline tilegx_bundle_bits -create_Imm8_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0xff)) << 43); -} - -static __inline tilegx_bundle_bits -create_Imm8_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xff) << 12); -} - -static __inline tilegx_bundle_bits -create_Imm8_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0xff)) << 43); -} - -static __inline tilegx_bundle_bits -create_JumpOff_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x7ffffff)) << 31); -} - -static __inline tilegx_bundle_bits -create_JumpOpcodeExtension_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x1)) << 58); -} - -static __inline tilegx_bundle_bits -create_MF_Imm14_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3fff)) << 37); -} - -static __inline tilegx_bundle_bits -create_MT_Imm14_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) | - (((tilegx_bundle_bits)(n & 0x00003fc0)) << 37); -} - -static __inline tilegx_bundle_bits -create_Mode(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3)) << 62); -} - -static __inline tilegx_bundle_bits -create_Opcode_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x7) << 28); -} - -static __inline tilegx_bundle_bits -create_Opcode_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x7)) << 59); -} - -static __inline tilegx_bundle_bits -create_Opcode_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0xf) << 27); -} - -static __inline tilegx_bundle_bits -create_Opcode_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0xf)) << 58); -} - -static __inline tilegx_bundle_bits -create_Opcode_Y2(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x00000001) << 26) | - (((tilegx_bundle_bits)(n & 0x00000002)) << 56); -} - -static __inline tilegx_bundle_bits -create_RRROpcodeExtension_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3ff) << 18); -} - -static __inline tilegx_bundle_bits -create_RRROpcodeExtension_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3ff)) << 49); -} - -static __inline tilegx_bundle_bits -create_RRROpcodeExtension_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3) << 18); -} - -static __inline tilegx_bundle_bits -create_RRROpcodeExtension_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3)) << 49); -} - -static __inline tilegx_bundle_bits -create_ShAmt_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_ShAmt_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -static __inline tilegx_bundle_bits -create_ShAmt_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_ShAmt_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -static __inline tilegx_bundle_bits -create_ShiftOpcodeExtension_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3ff) << 18); -} - -static __inline tilegx_bundle_bits -create_ShiftOpcodeExtension_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3ff)) << 49); -} - -static __inline tilegx_bundle_bits -create_ShiftOpcodeExtension_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3) << 18); -} - -static __inline tilegx_bundle_bits -create_ShiftOpcodeExtension_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3)) << 49); -} - -static __inline tilegx_bundle_bits -create_SrcA_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 6); -} - -static __inline tilegx_bundle_bits -create_SrcA_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 37); -} - -static __inline tilegx_bundle_bits -create_SrcA_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 6); -} - -static __inline tilegx_bundle_bits -create_SrcA_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 37); -} - -static __inline tilegx_bundle_bits -create_SrcA_Y2(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 20); -} - -static __inline tilegx_bundle_bits -create_SrcBDest_Y2(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 51); -} - -static __inline tilegx_bundle_bits -create_SrcB_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_SrcB_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -static __inline tilegx_bundle_bits -create_SrcB_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_SrcB_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -static __inline tilegx_bundle_bits -create_UnaryOpcodeExtension_X0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_UnaryOpcodeExtension_X1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -static __inline tilegx_bundle_bits -create_UnaryOpcodeExtension_Y0(int num) -{ - const unsigned int n = (unsigned int)num; - return ((n & 0x3f) << 12); -} - -static __inline tilegx_bundle_bits -create_UnaryOpcodeExtension_Y1(int num) -{ - const unsigned int n = (unsigned int)num; - return (((tilegx_bundle_bits)(n & 0x3f)) << 43); -} - -const struct tilegx_opcode tilegx_opcodes[336] = -{ - { "bpt", TILEGX_OPC_BPT, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffffffff80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a44ae00000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "info", TILEGX_OPC_INFO, 0xf, 1, TREG_ZERO, 1, - { { 0 }, { 1 }, { 2 }, { 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00fffULL, - 0xfff807ff80000000ULL, - 0x0000000078000fffULL, - 0x3c0007ff80000000ULL, - 0ULL - }, - { - 0x0000000040300fffULL, - 0x181807ff80000000ULL, - 0x0000000010000fffULL, - 0x0c0007ff80000000ULL, - -1ULL - } -#endif - }, - { "infol", TILEGX_OPC_INFOL, 0x3, 1, TREG_ZERO, 1, - { { 4 }, { 5 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc000000070000fffULL, - 0xf80007ff80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000070000fffULL, - 0x380007ff80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld4s_tls", TILEGX_OPC_LD4S_TLS, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1858000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld_tls", TILEGX_OPC_LD_TLS, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18a0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "move", TILEGX_OPC_MOVE, 0xf, 2, TREG_ZERO, 1, - { { 8, 9 }, { 6, 7 }, { 10, 11 }, { 12, 13 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0xfffff80000000000ULL, - 0x00000000780ff000ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - 0x000000005107f000ULL, - 0x283bf80000000000ULL, - 0x00000000500bf000ULL, - 0x2c05f80000000000ULL, - -1ULL - } -#endif - }, - { "movei", TILEGX_OPC_MOVEI, 0xf, 2, TREG_ZERO, 1, - { { 8, 0 }, { 6, 1 }, { 10, 2 }, { 12, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00fc0ULL, - 0xfff807e000000000ULL, - 0x0000000078000fc0ULL, - 0x3c0007e000000000ULL, - 0ULL - }, - { - 0x0000000040100fc0ULL, - 0x180807e000000000ULL, - 0x0000000000000fc0ULL, - 0x040007e000000000ULL, - -1ULL - } -#endif - }, - { "moveli", TILEGX_OPC_MOVELI, 0x3, 2, TREG_ZERO, 1, - { { 8, 4 }, { 6, 5 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc000000070000fc0ULL, - 0xf80007e000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000010000fc0ULL, - 0x000007e000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch", TILEGX_OPC_PREFETCH, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a801f80000000ULL, - -1ULL, - -1ULL, - 0x41f8000004000000ULL - } -#endif - }, - { "prefetch_add_l1", TILEGX_OPC_PREFETCH_ADD_L1, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1840001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_add_l1_fault", TILEGX_OPC_PREFETCH_ADD_L1_FAULT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1838001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_add_l2", TILEGX_OPC_PREFETCH_ADD_L2, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1850001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_add_l2_fault", TILEGX_OPC_PREFETCH_ADD_L2_FAULT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1848001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_add_l3", TILEGX_OPC_PREFETCH_ADD_L3, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1860001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_add_l3_fault", TILEGX_OPC_PREFETCH_ADD_L3_FAULT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8001f80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1858001f80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "prefetch_l1", TILEGX_OPC_PREFETCH_L1, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a801f80000000ULL, - -1ULL, - -1ULL, - 0x41f8000004000000ULL - } -#endif - }, - { "prefetch_l1_fault", TILEGX_OPC_PREFETCH_L1_FAULT, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a781f80000000ULL, - -1ULL, - -1ULL, - 0x41f8000000000000ULL - } -#endif - }, - { "prefetch_l2", TILEGX_OPC_PREFETCH_L2, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a901f80000000ULL, - -1ULL, - -1ULL, - 0x43f8000004000000ULL - } -#endif - }, - { "prefetch_l2_fault", TILEGX_OPC_PREFETCH_L2_FAULT, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a881f80000000ULL, - -1ULL, - -1ULL, - 0x43f8000000000000ULL - } -#endif - }, - { "prefetch_l3", TILEGX_OPC_PREFETCH_L3, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286aa01f80000000ULL, - -1ULL, - -1ULL, - 0x83f8000000000000ULL - } -#endif - }, - { "prefetch_l3_fault", TILEGX_OPC_PREFETCH_L3_FAULT, 0x12, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff81f80000000ULL, - 0ULL, - 0ULL, - 0xc3f8000004000000ULL - }, - { - -1ULL, - 0x286a981f80000000ULL, - -1ULL, - -1ULL, - 0x81f8000004000000ULL - } -#endif - }, - { "raise", TILEGX_OPC_RAISE, 0x2, 0, TREG_ZERO, 1, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffffffff80000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a44ae80000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "add", TILEGX_OPC_ADD, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000500c0000ULL, - 0x2806000000000000ULL, - 0x0000000028040000ULL, - 0x1802000000000000ULL, - -1ULL - } -#endif - }, - { "addi", TILEGX_OPC_ADDI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 10, 11, 2 }, { 12, 13, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0x0000000078000000ULL, - 0x3c00000000000000ULL, - 0ULL - }, - { - 0x0000000040100000ULL, - 0x1808000000000000ULL, - 0ULL, - 0x0400000000000000ULL, - -1ULL - } -#endif - }, - { "addli", TILEGX_OPC_ADDLI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 4 }, { 6, 7, 5 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc000000070000000ULL, - 0xf800000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000010000000ULL, - 0ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "addx", TILEGX_OPC_ADDX, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050080000ULL, - 0x2804000000000000ULL, - 0x0000000028000000ULL, - 0x1800000000000000ULL, - -1ULL - } -#endif - }, - { "addxi", TILEGX_OPC_ADDXI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 10, 11, 2 }, { 12, 13, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0x0000000078000000ULL, - 0x3c00000000000000ULL, - 0ULL - }, - { - 0x0000000040200000ULL, - 0x1810000000000000ULL, - 0x0000000008000000ULL, - 0x0800000000000000ULL, - -1ULL - } -#endif - }, - { "addxli", TILEGX_OPC_ADDXLI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 4 }, { 6, 7, 5 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc000000070000000ULL, - 0xf800000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000020000000ULL, - 0x0800000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "addxsc", TILEGX_OPC_ADDXSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050040000ULL, - 0x2802000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "and", TILEGX_OPC_AND, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050100000ULL, - 0x2808000000000000ULL, - 0x0000000050000000ULL, - 0x2c00000000000000ULL, - -1ULL - } -#endif - }, - { "andi", TILEGX_OPC_ANDI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 10, 11, 2 }, { 12, 13, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0x0000000078000000ULL, - 0x3c00000000000000ULL, - 0ULL - }, - { - 0x0000000040300000ULL, - 0x1818000000000000ULL, - 0x0000000010000000ULL, - 0x0c00000000000000ULL, - -1ULL - } -#endif - }, - { "beqz", TILEGX_OPC_BEQZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1440000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "beqzt", TILEGX_OPC_BEQZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1400000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bfexts", TILEGX_OPC_BFEXTS, 0x1, 4, TREG_ZERO, 1, - { { 8, 9, 21, 22 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007f000000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000034000000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bfextu", TILEGX_OPC_BFEXTU, 0x1, 4, TREG_ZERO, 1, - { { 8, 9, 21, 22 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007f000000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000035000000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bfins", TILEGX_OPC_BFINS, 0x1, 4, TREG_ZERO, 1, - { { 23, 9, 21, 22 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007f000000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000036000000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bgez", TILEGX_OPC_BGEZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x14c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bgezt", TILEGX_OPC_BGEZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1480000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bgtz", TILEGX_OPC_BGTZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1540000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bgtzt", TILEGX_OPC_BGTZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1500000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blbc", TILEGX_OPC_BLBC, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x15c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blbct", TILEGX_OPC_BLBCT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1580000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blbs", TILEGX_OPC_BLBS, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1640000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blbst", TILEGX_OPC_BLBST, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1600000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blez", TILEGX_OPC_BLEZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x16c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "blezt", TILEGX_OPC_BLEZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1680000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bltz", TILEGX_OPC_BLTZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1740000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bltzt", TILEGX_OPC_BLTZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1700000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bnez", TILEGX_OPC_BNEZ, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x17c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "bnezt", TILEGX_OPC_BNEZT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 20 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xffc0000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1780000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "clz", TILEGX_OPC_CLZ, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051481000ULL, - -1ULL, - 0x00000000300c1000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmoveqz", TILEGX_OPC_CMOVEQZ, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050140000ULL, - -1ULL, - 0x0000000048000000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmovnez", TILEGX_OPC_CMOVNEZ, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050180000ULL, - -1ULL, - 0x0000000048040000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmpeq", TILEGX_OPC_CMPEQ, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000501c0000ULL, - 0x280a000000000000ULL, - 0x0000000040000000ULL, - 0x2404000000000000ULL, - -1ULL - } -#endif - }, - { "cmpeqi", TILEGX_OPC_CMPEQI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 10, 11, 2 }, { 12, 13, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0x0000000078000000ULL, - 0x3c00000000000000ULL, - 0ULL - }, - { - 0x0000000040400000ULL, - 0x1820000000000000ULL, - 0x0000000018000000ULL, - 0x1000000000000000ULL, - -1ULL - } -#endif - }, - { "cmpexch", TILEGX_OPC_CMPEXCH, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x280e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmpexch4", TILEGX_OPC_CMPEXCH4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x280c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmples", TILEGX_OPC_CMPLES, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050200000ULL, - 0x2810000000000000ULL, - 0x0000000038000000ULL, - 0x2000000000000000ULL, - -1ULL - } -#endif - }, - { "cmpleu", TILEGX_OPC_CMPLEU, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050240000ULL, - 0x2812000000000000ULL, - 0x0000000038040000ULL, - 0x2002000000000000ULL, - -1ULL - } -#endif - }, - { "cmplts", TILEGX_OPC_CMPLTS, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050280000ULL, - 0x2814000000000000ULL, - 0x0000000038080000ULL, - 0x2004000000000000ULL, - -1ULL - } -#endif - }, - { "cmpltsi", TILEGX_OPC_CMPLTSI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 10, 11, 2 }, { 12, 13, 3 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0x0000000078000000ULL, - 0x3c00000000000000ULL, - 0ULL - }, - { - 0x0000000040500000ULL, - 0x1828000000000000ULL, - 0x0000000020000000ULL, - 0x1400000000000000ULL, - -1ULL - } -#endif - }, - { "cmpltu", TILEGX_OPC_CMPLTU, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000502c0000ULL, - 0x2816000000000000ULL, - 0x00000000380c0000ULL, - 0x2006000000000000ULL, - -1ULL - } -#endif - }, - { "cmpltui", TILEGX_OPC_CMPLTUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040600000ULL, - 0x1830000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmpne", TILEGX_OPC_CMPNE, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050300000ULL, - 0x2818000000000000ULL, - 0x0000000040040000ULL, - 0x2406000000000000ULL, - -1ULL - } -#endif - }, - { "cmul", TILEGX_OPC_CMUL, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000504c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmula", TILEGX_OPC_CMULA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050380000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmulaf", TILEGX_OPC_CMULAF, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050340000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmulf", TILEGX_OPC_CMULF, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050400000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmulfr", TILEGX_OPC_CMULFR, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000503c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmulh", TILEGX_OPC_CMULH, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050480000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "cmulhr", TILEGX_OPC_CMULHR, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050440000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "crc32_32", TILEGX_OPC_CRC32_32, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050500000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "crc32_8", TILEGX_OPC_CRC32_8, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050540000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ctz", TILEGX_OPC_CTZ, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051482000ULL, - -1ULL, - 0x00000000300c2000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "dblalign", TILEGX_OPC_DBLALIGN, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050640000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "dblalign2", TILEGX_OPC_DBLALIGN2, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050580000ULL, - 0x281a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "dblalign4", TILEGX_OPC_DBLALIGN4, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000505c0000ULL, - 0x281c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "dblalign6", TILEGX_OPC_DBLALIGN6, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050600000ULL, - 0x281e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "drain", TILEGX_OPC_DRAIN, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a080000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "dtlbpr", TILEGX_OPC_DTLBPR, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a100000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "exch", TILEGX_OPC_EXCH, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2822000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "exch4", TILEGX_OPC_EXCH4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2820000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_add_flags", TILEGX_OPC_FDOUBLE_ADD_FLAGS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000506c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_addsub", TILEGX_OPC_FDOUBLE_ADDSUB, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050680000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_mul_flags", TILEGX_OPC_FDOUBLE_MUL_FLAGS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050700000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_pack1", TILEGX_OPC_FDOUBLE_PACK1, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050740000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_pack2", TILEGX_OPC_FDOUBLE_PACK2, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050780000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_sub_flags", TILEGX_OPC_FDOUBLE_SUB_FLAGS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000507c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_unpack_max", TILEGX_OPC_FDOUBLE_UNPACK_MAX, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050800000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fdouble_unpack_min", TILEGX_OPC_FDOUBLE_UNPACK_MIN, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050840000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchadd", TILEGX_OPC_FETCHADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x282a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchadd4", TILEGX_OPC_FETCHADD4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2824000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchaddgez", TILEGX_OPC_FETCHADDGEZ, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2828000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchaddgez4", TILEGX_OPC_FETCHADDGEZ4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2826000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchand", TILEGX_OPC_FETCHAND, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x282e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchand4", TILEGX_OPC_FETCHAND4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x282c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchor", TILEGX_OPC_FETCHOR, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2832000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fetchor4", TILEGX_OPC_FETCHOR4, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2830000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "finv", TILEGX_OPC_FINV, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a180000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "flush", TILEGX_OPC_FLUSH, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a280000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "flushwb", TILEGX_OPC_FLUSHWB, 0x2, 0, TREG_ZERO, 1, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a200000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fnop", TILEGX_OPC_FNOP, 0xf, 0, TREG_ZERO, 1, - { { }, { }, { }, { }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0xfffff80000000000ULL, - 0x00000000780ff000ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - 0x0000000051483000ULL, - 0x286a300000000000ULL, - 0x00000000300c3000ULL, - 0x1c06400000000000ULL, - -1ULL - } -#endif - }, - { "fsingle_add1", TILEGX_OPC_FSINGLE_ADD1, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050880000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_addsub2", TILEGX_OPC_FSINGLE_ADDSUB2, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000508c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_mul1", TILEGX_OPC_FSINGLE_MUL1, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050900000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_mul2", TILEGX_OPC_FSINGLE_MUL2, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050940000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_pack1", TILEGX_OPC_FSINGLE_PACK1, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051484000ULL, - -1ULL, - 0x00000000300c4000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_pack2", TILEGX_OPC_FSINGLE_PACK2, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050980000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "fsingle_sub1", TILEGX_OPC_FSINGLE_SUB1, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000509c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "icoh", TILEGX_OPC_ICOH, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a380000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ill", TILEGX_OPC_ILL, 0xa, 0, TREG_ZERO, 1, - { { 0, }, { }, { 0, }, { }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286a400000000000ULL, - -1ULL, - 0x1c06480000000000ULL, - -1ULL - } -#endif - }, - { "inv", TILEGX_OPC_INV, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a480000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "iret", TILEGX_OPC_IRET, 0x2, 0, TREG_ZERO, 1, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286a500000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "j", TILEGX_OPC_J, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 25 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfc00000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2400000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "jal", TILEGX_OPC_JAL, 0x2, 1, TREG_LR, 1, - { { 0, }, { 25 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfc00000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2000000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "jalr", TILEGX_OPC_JALR, 0xa, 1, TREG_LR, 1, - { { 0, }, { 7 }, { 0, }, { 13 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286a600000000000ULL, - -1ULL, - 0x1c06580000000000ULL, - -1ULL - } -#endif - }, - { "jalrp", TILEGX_OPC_JALRP, 0xa, 1, TREG_LR, 1, - { { 0, }, { 7 }, { 0, }, { 13 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286a580000000000ULL, - -1ULL, - 0x1c06500000000000ULL, - -1ULL - } -#endif - }, - { "jr", TILEGX_OPC_JR, 0xa, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 13 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286a700000000000ULL, - -1ULL, - 0x1c06680000000000ULL, - -1ULL - } -#endif - }, - { "jrp", TILEGX_OPC_JRP, 0xa, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 13 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286a680000000000ULL, - -1ULL, - 0x1c06600000000000ULL, - -1ULL - } -#endif - }, - { "ld", TILEGX_OPC_LD, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286ae80000000000ULL, - -1ULL, - -1ULL, - 0x8200000004000000ULL - } -#endif - }, - { "ld1s", TILEGX_OPC_LD1S, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286a780000000000ULL, - -1ULL, - -1ULL, - 0x4000000000000000ULL - } -#endif - }, - { "ld1s_add", TILEGX_OPC_LD1S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1838000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld1u", TILEGX_OPC_LD1U, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286a800000000000ULL, - -1ULL, - -1ULL, - 0x4000000004000000ULL - } -#endif - }, - { "ld1u_add", TILEGX_OPC_LD1U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1840000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld2s", TILEGX_OPC_LD2S, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286a880000000000ULL, - -1ULL, - -1ULL, - 0x4200000000000000ULL - } -#endif - }, - { "ld2s_add", TILEGX_OPC_LD2S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1848000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld2u", TILEGX_OPC_LD2U, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286a900000000000ULL, - -1ULL, - -1ULL, - 0x4200000004000000ULL - } -#endif - }, - { "ld2u_add", TILEGX_OPC_LD2U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1850000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld4s", TILEGX_OPC_LD4S, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286a980000000000ULL, - -1ULL, - -1ULL, - 0x8000000004000000ULL - } -#endif - }, - { "ld4s_add", TILEGX_OPC_LD4S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1858000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld4u", TILEGX_OPC_LD4U, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 26, 14 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x286aa00000000000ULL, - -1ULL, - -1ULL, - 0x8200000000000000ULL - } -#endif - }, - { "ld4u_add", TILEGX_OPC_LD4U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1860000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ld_add", TILEGX_OPC_LD_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18a0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldna", TILEGX_OPC_LDNA, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286aa80000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldna_add", TILEGX_OPC_LDNA_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18a8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt", TILEGX_OPC_LDNT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ae00000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt1s", TILEGX_OPC_LDNT1S, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ab00000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt1s_add", TILEGX_OPC_LDNT1S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1868000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt1u", TILEGX_OPC_LDNT1U, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ab80000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt1u_add", TILEGX_OPC_LDNT1U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1870000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt2s", TILEGX_OPC_LDNT2S, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ac00000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt2s_add", TILEGX_OPC_LDNT2S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1878000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt2u", TILEGX_OPC_LDNT2U, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ac80000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt2u_add", TILEGX_OPC_LDNT2U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1880000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt4s", TILEGX_OPC_LDNT4S, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ad00000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt4s_add", TILEGX_OPC_LDNT4S_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1888000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt4u", TILEGX_OPC_LDNT4U, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286ad80000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt4u_add", TILEGX_OPC_LDNT4U_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1890000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "ldnt_add", TILEGX_OPC_LDNT_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 6, 15, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1898000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "lnk", TILEGX_OPC_LNK, 0xa, 1, TREG_ZERO, 1, - { { 0, }, { 6 }, { 0, }, { 12 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - -1ULL, - 0x286af00000000000ULL, - -1ULL, - 0x1c06700000000000ULL, - -1ULL - } -#endif - }, - { "mf", TILEGX_OPC_MF, 0x2, 0, TREG_ZERO, 1, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286af80000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mfspr", TILEGX_OPC_MFSPR, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 6, 27 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18b0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mm", TILEGX_OPC_MM, 0x1, 4, TREG_ZERO, 1, - { { 23, 9, 21, 22 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007f000000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000037000000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mnz", TILEGX_OPC_MNZ, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050a00000ULL, - 0x2834000000000000ULL, - 0x0000000048080000ULL, - 0x2804000000000000ULL, - -1ULL - } -#endif - }, - { "mtspr", TILEGX_OPC_MTSPR, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 28, 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18b8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hs_hs", TILEGX_OPC_MUL_HS_HS, 0x5, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 10, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050d40000ULL, - -1ULL, - 0x0000000068000000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hs_hu", TILEGX_OPC_MUL_HS_HU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050d80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hs_ls", TILEGX_OPC_MUL_HS_LS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050dc0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hs_lu", TILEGX_OPC_MUL_HS_LU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050e00000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hu_hu", TILEGX_OPC_MUL_HU_HU, 0x5, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 10, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050e40000ULL, - -1ULL, - 0x0000000068040000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hu_ls", TILEGX_OPC_MUL_HU_LS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050e80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_hu_lu", TILEGX_OPC_MUL_HU_LU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050ec0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_ls_ls", TILEGX_OPC_MUL_LS_LS, 0x5, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 10, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050f00000ULL, - -1ULL, - 0x0000000068080000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_ls_lu", TILEGX_OPC_MUL_LS_LU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050f40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mul_lu_lu", TILEGX_OPC_MUL_LU_LU, 0x5, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 10, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050f80000ULL, - -1ULL, - 0x00000000680c0000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hs_hs", TILEGX_OPC_MULA_HS_HS, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050a80000ULL, - -1ULL, - 0x0000000070000000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hs_hu", TILEGX_OPC_MULA_HS_HU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050ac0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hs_ls", TILEGX_OPC_MULA_HS_LS, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050b00000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hs_lu", TILEGX_OPC_MULA_HS_LU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050b40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hu_hu", TILEGX_OPC_MULA_HU_HU, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050b80000ULL, - -1ULL, - 0x0000000070040000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hu_ls", TILEGX_OPC_MULA_HU_LS, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050bc0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_hu_lu", TILEGX_OPC_MULA_HU_LU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050c00000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_ls_ls", TILEGX_OPC_MULA_LS_LS, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050c40000ULL, - -1ULL, - 0x0000000070080000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_ls_lu", TILEGX_OPC_MULA_LS_LU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050c80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mula_lu_lu", TILEGX_OPC_MULA_LU_LU, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050cc0000ULL, - -1ULL, - 0x00000000700c0000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mulax", TILEGX_OPC_MULAX, 0x5, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 24, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050a40000ULL, - -1ULL, - 0x0000000040080000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mulx", TILEGX_OPC_MULX, 0x5, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 10, 11, 18 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0x00000000780c0000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000050d00000ULL, - -1ULL, - 0x00000000400c0000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "mz", TILEGX_OPC_MZ, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000050fc0000ULL, - 0x2836000000000000ULL, - 0x00000000480c0000ULL, - 0x2806000000000000ULL, - -1ULL - } -#endif - }, - { "nap", TILEGX_OPC_NAP, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "nop", TILEGX_OPC_NOP, 0xf, 0, TREG_ZERO, 1, - { { }, { }, { }, { }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0xfffff80000000000ULL, - 0x00000000780ff000ULL, - 0x3c07f80000000000ULL, - 0ULL - }, - { - 0x0000000051485000ULL, - 0x286b080000000000ULL, - 0x00000000300c5000ULL, - 0x1c06780000000000ULL, - -1ULL - } -#endif - }, - { "nor", TILEGX_OPC_NOR, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051000000ULL, - 0x2838000000000000ULL, - 0x0000000050040000ULL, - 0x2c02000000000000ULL, - -1ULL - } -#endif - }, - { "or", TILEGX_OPC_OR, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051040000ULL, - 0x283a000000000000ULL, - 0x0000000050080000ULL, - 0x2c04000000000000ULL, - -1ULL - } -#endif - }, - { "ori", TILEGX_OPC_ORI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040700000ULL, - 0x18c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "pcnt", TILEGX_OPC_PCNT, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051486000ULL, - -1ULL, - 0x00000000300c6000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "revbits", TILEGX_OPC_REVBITS, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051487000ULL, - -1ULL, - 0x00000000300c7000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "revbytes", TILEGX_OPC_REVBYTES, 0x5, 2, TREG_ZERO, 1, - { { 8, 9 }, { 0, }, { 10, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051488000ULL, - -1ULL, - 0x00000000300c8000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "rotl", TILEGX_OPC_ROTL, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051080000ULL, - 0x283c000000000000ULL, - 0x0000000058000000ULL, - 0x3000000000000000ULL, - -1ULL - } -#endif - }, - { "rotli", TILEGX_OPC_ROTLI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 10, 11, 31 }, { 12, 13, 32 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000060040000ULL, - 0x3002000000000000ULL, - 0x0000000078000000ULL, - 0x3800000000000000ULL, - -1ULL - } -#endif - }, - { "shl", TILEGX_OPC_SHL, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051280000ULL, - 0x284c000000000000ULL, - 0x0000000058040000ULL, - 0x3002000000000000ULL, - -1ULL - } -#endif - }, - { "shl16insli", TILEGX_OPC_SHL16INSLI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 4 }, { 6, 7, 5 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc000000070000000ULL, - 0xf800000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000070000000ULL, - 0x3800000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "shl1add", TILEGX_OPC_SHL1ADD, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051100000ULL, - 0x2840000000000000ULL, - 0x0000000030000000ULL, - 0x1c00000000000000ULL, - -1ULL - } -#endif - }, - { "shl1addx", TILEGX_OPC_SHL1ADDX, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000510c0000ULL, - 0x283e000000000000ULL, - 0x0000000060040000ULL, - 0x3402000000000000ULL, - -1ULL - } -#endif - }, - { "shl2add", TILEGX_OPC_SHL2ADD, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051180000ULL, - 0x2844000000000000ULL, - 0x0000000030040000ULL, - 0x1c02000000000000ULL, - -1ULL - } -#endif - }, - { "shl2addx", TILEGX_OPC_SHL2ADDX, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051140000ULL, - 0x2842000000000000ULL, - 0x0000000060080000ULL, - 0x3404000000000000ULL, - -1ULL - } -#endif - }, - { "shl3add", TILEGX_OPC_SHL3ADD, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051200000ULL, - 0x2848000000000000ULL, - 0x0000000030080000ULL, - 0x1c04000000000000ULL, - -1ULL - } -#endif - }, - { "shl3addx", TILEGX_OPC_SHL3ADDX, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000511c0000ULL, - 0x2846000000000000ULL, - 0x00000000600c0000ULL, - 0x3406000000000000ULL, - -1ULL - } -#endif - }, - { "shli", TILEGX_OPC_SHLI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 10, 11, 31 }, { 12, 13, 32 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000060080000ULL, - 0x3004000000000000ULL, - 0x0000000078040000ULL, - 0x3802000000000000ULL, - -1ULL - } -#endif - }, - { "shlx", TILEGX_OPC_SHLX, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051240000ULL, - 0x284a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "shlxi", TILEGX_OPC_SHLXI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000600c0000ULL, - 0x3006000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "shrs", TILEGX_OPC_SHRS, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x00000000512c0000ULL, - 0x284e000000000000ULL, - 0x0000000058080000ULL, - 0x3004000000000000ULL, - -1ULL - } -#endif - }, - { "shrsi", TILEGX_OPC_SHRSI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 10, 11, 31 }, { 12, 13, 32 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000060100000ULL, - 0x3008000000000000ULL, - 0x0000000078080000ULL, - 0x3804000000000000ULL, - -1ULL - } -#endif - }, - { "shru", TILEGX_OPC_SHRU, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051340000ULL, - 0x2852000000000000ULL, - 0x00000000580c0000ULL, - 0x3006000000000000ULL, - -1ULL - } -#endif - }, - { "shrui", TILEGX_OPC_SHRUI, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 10, 11, 31 }, { 12, 13, 32 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000060140000ULL, - 0x300a000000000000ULL, - 0x00000000780c0000ULL, - 0x3806000000000000ULL, - -1ULL - } -#endif - }, - { "shrux", TILEGX_OPC_SHRUX, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051300000ULL, - 0x2850000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "shruxi", TILEGX_OPC_SHRUXI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000060180000ULL, - 0x300c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "shufflebytes", TILEGX_OPC_SHUFFLEBYTES, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051380000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "st", TILEGX_OPC_ST, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 14, 33 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x2862000000000000ULL, - -1ULL, - -1ULL, - 0xc200000004000000ULL - } -#endif - }, - { "st1", TILEGX_OPC_ST1, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 14, 33 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x2854000000000000ULL, - -1ULL, - -1ULL, - 0xc000000000000000ULL - } -#endif - }, - { "st1_add", TILEGX_OPC_ST1_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18c8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "st2", TILEGX_OPC_ST2, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 14, 33 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x2856000000000000ULL, - -1ULL, - -1ULL, - 0xc000000004000000ULL - } -#endif - }, - { "st2_add", TILEGX_OPC_ST2_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18d0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "st4", TILEGX_OPC_ST4, 0x12, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 14, 33 } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0xc200000004000000ULL - }, - { - -1ULL, - 0x2858000000000000ULL, - -1ULL, - -1ULL, - 0xc200000000000000ULL - } -#endif - }, - { "st4_add", TILEGX_OPC_ST4_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18d8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "st_add", TILEGX_OPC_ST_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x1900000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt", TILEGX_OPC_STNT, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x2860000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt1", TILEGX_OPC_STNT1, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x285a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt1_add", TILEGX_OPC_STNT1_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18e0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt2", TILEGX_OPC_STNT2, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x285c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt2_add", TILEGX_OPC_STNT2_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18e8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt4", TILEGX_OPC_STNT4, 0x2, 2, TREG_ZERO, 1, - { { 0, }, { 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x285e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt4_add", TILEGX_OPC_STNT4_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18f0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "stnt_add", TILEGX_OPC_STNT_ADD, 0x2, 3, TREG_ZERO, 1, - { { 0, }, { 15, 17, 34 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x18f8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "sub", TILEGX_OPC_SUB, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051440000ULL, - 0x2868000000000000ULL, - 0x00000000280c0000ULL, - 0x1806000000000000ULL, - -1ULL - } -#endif - }, - { "subx", TILEGX_OPC_SUBX, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000051400000ULL, - 0x2866000000000000ULL, - 0x0000000028080000ULL, - 0x1804000000000000ULL, - -1ULL - } -#endif - }, - { "subxsc", TILEGX_OPC_SUBXSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000513c0000ULL, - 0x2864000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "swint0", TILEGX_OPC_SWINT0, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b100000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "swint1", TILEGX_OPC_SWINT1, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b180000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "swint2", TILEGX_OPC_SWINT2, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b200000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "swint3", TILEGX_OPC_SWINT3, 0x2, 0, TREG_ZERO, 0, - { { 0, }, { }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b280000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "tblidxb0", TILEGX_OPC_TBLIDXB0, 0x5, 2, TREG_ZERO, 1, - { { 23, 9 }, { 0, }, { 24, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051489000ULL, - -1ULL, - 0x00000000300c9000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "tblidxb1", TILEGX_OPC_TBLIDXB1, 0x5, 2, TREG_ZERO, 1, - { { 23, 9 }, { 0, }, { 24, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x000000005148a000ULL, - -1ULL, - 0x00000000300ca000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "tblidxb2", TILEGX_OPC_TBLIDXB2, 0x5, 2, TREG_ZERO, 1, - { { 23, 9 }, { 0, }, { 24, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x000000005148b000ULL, - -1ULL, - 0x00000000300cb000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "tblidxb3", TILEGX_OPC_TBLIDXB3, 0x5, 2, TREG_ZERO, 1, - { { 23, 9 }, { 0, }, { 24, 11 }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffff000ULL, - 0ULL, - 0x00000000780ff000ULL, - 0ULL, - 0ULL - }, - { - 0x000000005148c000ULL, - -1ULL, - 0x00000000300cc000ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1add", TILEGX_OPC_V1ADD, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051500000ULL, - 0x286e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1addi", TILEGX_OPC_V1ADDI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040800000ULL, - 0x1908000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1adduc", TILEGX_OPC_V1ADDUC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000514c0000ULL, - 0x286c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1adiffu", TILEGX_OPC_V1ADIFFU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051540000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1avgu", TILEGX_OPC_V1AVGU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051580000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpeq", TILEGX_OPC_V1CMPEQ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000515c0000ULL, - 0x2870000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpeqi", TILEGX_OPC_V1CMPEQI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040900000ULL, - 0x1910000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmples", TILEGX_OPC_V1CMPLES, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051600000ULL, - 0x2872000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpleu", TILEGX_OPC_V1CMPLEU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051640000ULL, - 0x2874000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmplts", TILEGX_OPC_V1CMPLTS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051680000ULL, - 0x2876000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpltsi", TILEGX_OPC_V1CMPLTSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040a00000ULL, - 0x1918000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpltu", TILEGX_OPC_V1CMPLTU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000516c0000ULL, - 0x2878000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpltui", TILEGX_OPC_V1CMPLTUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040b00000ULL, - 0x1920000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1cmpne", TILEGX_OPC_V1CMPNE, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051700000ULL, - 0x287a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1ddotpu", TILEGX_OPC_V1DDOTPU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052880000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1ddotpua", TILEGX_OPC_V1DDOTPUA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052840000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1ddotpus", TILEGX_OPC_V1DDOTPUS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051780000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1ddotpusa", TILEGX_OPC_V1DDOTPUSA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051740000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotp", TILEGX_OPC_V1DOTP, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051880000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotpa", TILEGX_OPC_V1DOTPA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000517c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotpu", TILEGX_OPC_V1DOTPU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052900000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotpua", TILEGX_OPC_V1DOTPUA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000528c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotpus", TILEGX_OPC_V1DOTPUS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051840000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1dotpusa", TILEGX_OPC_V1DOTPUSA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051800000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1int_h", TILEGX_OPC_V1INT_H, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000518c0000ULL, - 0x287c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1int_l", TILEGX_OPC_V1INT_L, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051900000ULL, - 0x287e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1maxu", TILEGX_OPC_V1MAXU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051940000ULL, - 0x2880000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1maxui", TILEGX_OPC_V1MAXUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040c00000ULL, - 0x1928000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1minu", TILEGX_OPC_V1MINU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051980000ULL, - 0x2882000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1minui", TILEGX_OPC_V1MINUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040d00000ULL, - 0x1930000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1mnz", TILEGX_OPC_V1MNZ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000519c0000ULL, - 0x2884000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1multu", TILEGX_OPC_V1MULTU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051a00000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1mulu", TILEGX_OPC_V1MULU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051a80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1mulus", TILEGX_OPC_V1MULUS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051a40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1mz", TILEGX_OPC_V1MZ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051ac0000ULL, - 0x2886000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1sadau", TILEGX_OPC_V1SADAU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051b00000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1sadu", TILEGX_OPC_V1SADU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051b40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shl", TILEGX_OPC_V1SHL, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051b80000ULL, - 0x2888000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shli", TILEGX_OPC_V1SHLI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000601c0000ULL, - 0x300e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shrs", TILEGX_OPC_V1SHRS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051bc0000ULL, - 0x288a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shrsi", TILEGX_OPC_V1SHRSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000060200000ULL, - 0x3010000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shru", TILEGX_OPC_V1SHRU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051c00000ULL, - 0x288c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1shrui", TILEGX_OPC_V1SHRUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000060240000ULL, - 0x3012000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1sub", TILEGX_OPC_V1SUB, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051c80000ULL, - 0x2890000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v1subuc", TILEGX_OPC_V1SUBUC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051c40000ULL, - 0x288e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2add", TILEGX_OPC_V2ADD, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051d00000ULL, - 0x2894000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2addi", TILEGX_OPC_V2ADDI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040e00000ULL, - 0x1938000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2addsc", TILEGX_OPC_V2ADDSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051cc0000ULL, - 0x2892000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2adiffs", TILEGX_OPC_V2ADIFFS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051d40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2avgs", TILEGX_OPC_V2AVGS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051d80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpeq", TILEGX_OPC_V2CMPEQ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051dc0000ULL, - 0x2896000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpeqi", TILEGX_OPC_V2CMPEQI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000040f00000ULL, - 0x1940000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmples", TILEGX_OPC_V2CMPLES, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051e00000ULL, - 0x2898000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpleu", TILEGX_OPC_V2CMPLEU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051e40000ULL, - 0x289a000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmplts", TILEGX_OPC_V2CMPLTS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051e80000ULL, - 0x289c000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpltsi", TILEGX_OPC_V2CMPLTSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000041000000ULL, - 0x1948000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpltu", TILEGX_OPC_V2CMPLTU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051ec0000ULL, - 0x289e000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpltui", TILEGX_OPC_V2CMPLTUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000041100000ULL, - 0x1950000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2cmpne", TILEGX_OPC_V2CMPNE, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051f00000ULL, - 0x28a0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2dotp", TILEGX_OPC_V2DOTP, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051f80000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2dotpa", TILEGX_OPC_V2DOTPA, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051f40000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2int_h", TILEGX_OPC_V2INT_H, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000051fc0000ULL, - 0x28a2000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2int_l", TILEGX_OPC_V2INT_L, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052000000ULL, - 0x28a4000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2maxs", TILEGX_OPC_V2MAXS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052040000ULL, - 0x28a6000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2maxsi", TILEGX_OPC_V2MAXSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000041200000ULL, - 0x1958000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2mins", TILEGX_OPC_V2MINS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052080000ULL, - 0x28a8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2minsi", TILEGX_OPC_V2MINSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000041300000ULL, - 0x1960000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2mnz", TILEGX_OPC_V2MNZ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000520c0000ULL, - 0x28aa000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2mulfsc", TILEGX_OPC_V2MULFSC, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052100000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2muls", TILEGX_OPC_V2MULS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052140000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2mults", TILEGX_OPC_V2MULTS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052180000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2mz", TILEGX_OPC_V2MZ, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000521c0000ULL, - 0x28ac000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2packh", TILEGX_OPC_V2PACKH, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052200000ULL, - 0x28ae000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2packl", TILEGX_OPC_V2PACKL, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052240000ULL, - 0x28b0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2packuc", TILEGX_OPC_V2PACKUC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052280000ULL, - 0x28b2000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2sadas", TILEGX_OPC_V2SADAS, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000522c0000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2sadau", TILEGX_OPC_V2SADAU, 0x1, 3, TREG_ZERO, 1, - { { 23, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052300000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2sads", TILEGX_OPC_V2SADS, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052340000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2sadu", TILEGX_OPC_V2SADU, 0x1, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 0, }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052380000ULL, - -1ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shl", TILEGX_OPC_V2SHL, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052400000ULL, - 0x28b6000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shli", TILEGX_OPC_V2SHLI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000060280000ULL, - 0x3014000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shlsc", TILEGX_OPC_V2SHLSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000523c0000ULL, - 0x28b4000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shrs", TILEGX_OPC_V2SHRS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052440000ULL, - 0x28b8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shrsi", TILEGX_OPC_V2SHRSI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000602c0000ULL, - 0x3016000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shru", TILEGX_OPC_V2SHRU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052480000ULL, - 0x28ba000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2shrui", TILEGX_OPC_V2SHRUI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 29 }, { 6, 7, 30 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000060300000ULL, - 0x3018000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2sub", TILEGX_OPC_V2SUB, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052500000ULL, - 0x28be000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v2subsc", TILEGX_OPC_V2SUBSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000524c0000ULL, - 0x28bc000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4add", TILEGX_OPC_V4ADD, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052580000ULL, - 0x28c2000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4addsc", TILEGX_OPC_V4ADDSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052540000ULL, - 0x28c0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4int_h", TILEGX_OPC_V4INT_H, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000525c0000ULL, - 0x28c4000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4int_l", TILEGX_OPC_V4INT_L, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052600000ULL, - 0x28c6000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4packsc", TILEGX_OPC_V4PACKSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052640000ULL, - 0x28c8000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4shl", TILEGX_OPC_V4SHL, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000526c0000ULL, - 0x28cc000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4shlsc", TILEGX_OPC_V4SHLSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052680000ULL, - 0x28ca000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4shrs", TILEGX_OPC_V4SHRS, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052700000ULL, - 0x28ce000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4shru", TILEGX_OPC_V4SHRU, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052740000ULL, - 0x28d0000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4sub", TILEGX_OPC_V4SUB, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x00000000527c0000ULL, - 0x28d4000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "v4subsc", TILEGX_OPC_V4SUBSC, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000052780000ULL, - 0x28d2000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "wh64", TILEGX_OPC_WH64, 0x2, 1, TREG_ZERO, 1, - { { 0, }, { 7 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0ULL, - 0xfffff80000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - -1ULL, - 0x286b300000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { "xor", TILEGX_OPC_XOR, 0xf, 3, TREG_ZERO, 1, - { { 8, 9, 16 }, { 6, 7, 17 }, { 10, 11, 18 }, { 12, 13, 19 }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ffc0000ULL, - 0xfffe000000000000ULL, - 0x00000000780c0000ULL, - 0x3c06000000000000ULL, - 0ULL - }, - { - 0x0000000052800000ULL, - 0x28d6000000000000ULL, - 0x00000000500c0000ULL, - 0x2c06000000000000ULL, - -1ULL - } -#endif - }, - { "xori", TILEGX_OPC_XORI, 0x3, 3, TREG_ZERO, 1, - { { 8, 9, 0 }, { 6, 7, 1 }, { 0, }, { 0, }, { 0, } }, -#ifndef DISASM_ONLY - { - 0xc00000007ff00000ULL, - 0xfff8000000000000ULL, - 0ULL, - 0ULL, - 0ULL - }, - { - 0x0000000041400000ULL, - 0x1968000000000000ULL, - -1ULL, - -1ULL, - -1ULL - } -#endif - }, - { NULL, TILEGX_OPC_NONE, 0, 0, TREG_ZERO, 0, { { 0, } }, -#ifndef DISASM_ONLY - { 0, }, { 0, } -#endif - } -}; - -#define BITFIELD(start, size) ((start) | (((1 << (size)) - 1) << 6)) -#define CHILD(array_index) (TILEGX_OPC_NONE + (array_index)) - -static const unsigned short decode_X0_fsm[936] = -{ - BITFIELD(22, 9) /* index 0 */, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_BFEXTS, - TILEGX_OPC_BFEXTS, TILEGX_OPC_BFEXTS, TILEGX_OPC_BFEXTS, TILEGX_OPC_BFEXTU, - TILEGX_OPC_BFEXTU, TILEGX_OPC_BFEXTU, TILEGX_OPC_BFEXTU, TILEGX_OPC_BFINS, - TILEGX_OPC_BFINS, TILEGX_OPC_BFINS, TILEGX_OPC_BFINS, TILEGX_OPC_MM, - TILEGX_OPC_MM, TILEGX_OPC_MM, TILEGX_OPC_MM, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, CHILD(528), CHILD(578), - CHILD(583), CHILD(588), CHILD(593), CHILD(598), TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, CHILD(603), CHILD(620), CHILD(637), CHILD(654), CHILD(671), - CHILD(703), CHILD(797), CHILD(814), CHILD(831), CHILD(848), CHILD(865), - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, CHILD(889), TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - CHILD(906), CHILD(906), CHILD(906), CHILD(906), CHILD(906), - BITFIELD(6, 2) /* index 513 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, CHILD(518), - BITFIELD(8, 2) /* index 518 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, CHILD(523), - BITFIELD(10, 2) /* index 523 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_MOVELI, - BITFIELD(20, 2) /* index 528 */, - TILEGX_OPC_NONE, CHILD(533), TILEGX_OPC_ADDXI, CHILD(548), - BITFIELD(6, 2) /* index 533 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(538), - BITFIELD(8, 2) /* index 538 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(543), - BITFIELD(10, 2) /* index 543 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_MOVEI, - BITFIELD(0, 2) /* index 548 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(553), - BITFIELD(2, 2) /* index 553 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(558), - BITFIELD(4, 2) /* index 558 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(563), - BITFIELD(6, 2) /* index 563 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(568), - BITFIELD(8, 2) /* index 568 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(573), - BITFIELD(10, 2) /* index 573 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_INFO, - BITFIELD(20, 2) /* index 578 */, - TILEGX_OPC_CMPEQI, TILEGX_OPC_CMPLTSI, TILEGX_OPC_CMPLTUI, TILEGX_OPC_ORI, - BITFIELD(20, 2) /* index 583 */, - TILEGX_OPC_V1ADDI, TILEGX_OPC_V1CMPEQI, TILEGX_OPC_V1CMPLTSI, - TILEGX_OPC_V1CMPLTUI, - BITFIELD(20, 2) /* index 588 */, - TILEGX_OPC_V1MAXUI, TILEGX_OPC_V1MINUI, TILEGX_OPC_V2ADDI, - TILEGX_OPC_V2CMPEQI, - BITFIELD(20, 2) /* index 593 */, - TILEGX_OPC_V2CMPLTSI, TILEGX_OPC_V2CMPLTUI, TILEGX_OPC_V2MAXSI, - TILEGX_OPC_V2MINSI, - BITFIELD(20, 2) /* index 598 */, - TILEGX_OPC_XORI, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(18, 4) /* index 603 */, - TILEGX_OPC_NONE, TILEGX_OPC_ADDXSC, TILEGX_OPC_ADDX, TILEGX_OPC_ADD, - TILEGX_OPC_AND, TILEGX_OPC_CMOVEQZ, TILEGX_OPC_CMOVNEZ, TILEGX_OPC_CMPEQ, - TILEGX_OPC_CMPLES, TILEGX_OPC_CMPLEU, TILEGX_OPC_CMPLTS, TILEGX_OPC_CMPLTU, - TILEGX_OPC_CMPNE, TILEGX_OPC_CMULAF, TILEGX_OPC_CMULA, TILEGX_OPC_CMULFR, - BITFIELD(18, 4) /* index 620 */, - TILEGX_OPC_CMULF, TILEGX_OPC_CMULHR, TILEGX_OPC_CMULH, TILEGX_OPC_CMUL, - TILEGX_OPC_CRC32_32, TILEGX_OPC_CRC32_8, TILEGX_OPC_DBLALIGN2, - TILEGX_OPC_DBLALIGN4, TILEGX_OPC_DBLALIGN6, TILEGX_OPC_DBLALIGN, - TILEGX_OPC_FDOUBLE_ADDSUB, TILEGX_OPC_FDOUBLE_ADD_FLAGS, - TILEGX_OPC_FDOUBLE_MUL_FLAGS, TILEGX_OPC_FDOUBLE_PACK1, - TILEGX_OPC_FDOUBLE_PACK2, TILEGX_OPC_FDOUBLE_SUB_FLAGS, - BITFIELD(18, 4) /* index 637 */, - TILEGX_OPC_FDOUBLE_UNPACK_MAX, TILEGX_OPC_FDOUBLE_UNPACK_MIN, - TILEGX_OPC_FSINGLE_ADD1, TILEGX_OPC_FSINGLE_ADDSUB2, - TILEGX_OPC_FSINGLE_MUL1, TILEGX_OPC_FSINGLE_MUL2, TILEGX_OPC_FSINGLE_PACK2, - TILEGX_OPC_FSINGLE_SUB1, TILEGX_OPC_MNZ, TILEGX_OPC_MULAX, - TILEGX_OPC_MULA_HS_HS, TILEGX_OPC_MULA_HS_HU, TILEGX_OPC_MULA_HS_LS, - TILEGX_OPC_MULA_HS_LU, TILEGX_OPC_MULA_HU_HU, TILEGX_OPC_MULA_HU_LS, - BITFIELD(18, 4) /* index 654 */, - TILEGX_OPC_MULA_HU_LU, TILEGX_OPC_MULA_LS_LS, TILEGX_OPC_MULA_LS_LU, - TILEGX_OPC_MULA_LU_LU, TILEGX_OPC_MULX, TILEGX_OPC_MUL_HS_HS, - TILEGX_OPC_MUL_HS_HU, TILEGX_OPC_MUL_HS_LS, TILEGX_OPC_MUL_HS_LU, - TILEGX_OPC_MUL_HU_HU, TILEGX_OPC_MUL_HU_LS, TILEGX_OPC_MUL_HU_LU, - TILEGX_OPC_MUL_LS_LS, TILEGX_OPC_MUL_LS_LU, TILEGX_OPC_MUL_LU_LU, - TILEGX_OPC_MZ, - BITFIELD(18, 4) /* index 671 */, - TILEGX_OPC_NOR, CHILD(688), TILEGX_OPC_ROTL, TILEGX_OPC_SHL1ADDX, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL2ADDX, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL3ADDX, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHLX, TILEGX_OPC_SHL, - TILEGX_OPC_SHRS, TILEGX_OPC_SHRUX, TILEGX_OPC_SHRU, TILEGX_OPC_SHUFFLEBYTES, - TILEGX_OPC_SUBXSC, - BITFIELD(12, 2) /* index 688 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(693), - BITFIELD(14, 2) /* index 693 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(698), - BITFIELD(16, 2) /* index 698 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_MOVE, - BITFIELD(18, 4) /* index 703 */, - TILEGX_OPC_SUBX, TILEGX_OPC_SUB, CHILD(720), TILEGX_OPC_V1ADDUC, - TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADIFFU, TILEGX_OPC_V1AVGU, - TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLEU, - TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPNE, - TILEGX_OPC_V1DDOTPUSA, TILEGX_OPC_V1DDOTPUS, TILEGX_OPC_V1DOTPA, - BITFIELD(12, 4) /* index 720 */, - TILEGX_OPC_NONE, CHILD(737), CHILD(742), CHILD(747), CHILD(752), CHILD(757), - CHILD(762), CHILD(767), CHILD(772), CHILD(777), CHILD(782), CHILD(787), - CHILD(792), TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 737 */, - TILEGX_OPC_CLZ, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 742 */, - TILEGX_OPC_CTZ, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 747 */, - TILEGX_OPC_FNOP, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 752 */, - TILEGX_OPC_FSINGLE_PACK1, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 757 */, - TILEGX_OPC_NOP, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 762 */, - TILEGX_OPC_PCNT, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 767 */, - TILEGX_OPC_REVBITS, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 772 */, - TILEGX_OPC_REVBYTES, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 777 */, - TILEGX_OPC_TBLIDXB0, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 782 */, - TILEGX_OPC_TBLIDXB1, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 787 */, - TILEGX_OPC_TBLIDXB2, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(16, 2) /* index 792 */, - TILEGX_OPC_TBLIDXB3, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(18, 4) /* index 797 */, - TILEGX_OPC_V1DOTPUSA, TILEGX_OPC_V1DOTPUS, TILEGX_OPC_V1DOTP, - TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_L, TILEGX_OPC_V1MAXU, - TILEGX_OPC_V1MINU, TILEGX_OPC_V1MNZ, TILEGX_OPC_V1MULTU, TILEGX_OPC_V1MULUS, - TILEGX_OPC_V1MULU, TILEGX_OPC_V1MZ, TILEGX_OPC_V1SADAU, TILEGX_OPC_V1SADU, - TILEGX_OPC_V1SHL, TILEGX_OPC_V1SHRS, - BITFIELD(18, 4) /* index 814 */, - TILEGX_OPC_V1SHRU, TILEGX_OPC_V1SUBUC, TILEGX_OPC_V1SUB, TILEGX_OPC_V2ADDSC, - TILEGX_OPC_V2ADD, TILEGX_OPC_V2ADIFFS, TILEGX_OPC_V2AVGS, - TILEGX_OPC_V2CMPEQ, TILEGX_OPC_V2CMPLES, TILEGX_OPC_V2CMPLEU, - TILEGX_OPC_V2CMPLTS, TILEGX_OPC_V2CMPLTU, TILEGX_OPC_V2CMPNE, - TILEGX_OPC_V2DOTPA, TILEGX_OPC_V2DOTP, TILEGX_OPC_V2INT_H, - BITFIELD(18, 4) /* index 831 */, - TILEGX_OPC_V2INT_L, TILEGX_OPC_V2MAXS, TILEGX_OPC_V2MINS, TILEGX_OPC_V2MNZ, - TILEGX_OPC_V2MULFSC, TILEGX_OPC_V2MULS, TILEGX_OPC_V2MULTS, TILEGX_OPC_V2MZ, - TILEGX_OPC_V2PACKH, TILEGX_OPC_V2PACKL, TILEGX_OPC_V2PACKUC, - TILEGX_OPC_V2SADAS, TILEGX_OPC_V2SADAU, TILEGX_OPC_V2SADS, - TILEGX_OPC_V2SADU, TILEGX_OPC_V2SHLSC, - BITFIELD(18, 4) /* index 848 */, - TILEGX_OPC_V2SHL, TILEGX_OPC_V2SHRS, TILEGX_OPC_V2SHRU, TILEGX_OPC_V2SUBSC, - TILEGX_OPC_V2SUB, TILEGX_OPC_V4ADDSC, TILEGX_OPC_V4ADD, TILEGX_OPC_V4INT_H, - TILEGX_OPC_V4INT_L, TILEGX_OPC_V4PACKSC, TILEGX_OPC_V4SHLSC, - TILEGX_OPC_V4SHL, TILEGX_OPC_V4SHRS, TILEGX_OPC_V4SHRU, TILEGX_OPC_V4SUBSC, - TILEGX_OPC_V4SUB, - BITFIELD(18, 3) /* index 865 */, - CHILD(874), CHILD(877), CHILD(880), CHILD(883), CHILD(886), TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(21, 1) /* index 874 */, - TILEGX_OPC_XOR, TILEGX_OPC_NONE, - BITFIELD(21, 1) /* index 877 */, - TILEGX_OPC_V1DDOTPUA, TILEGX_OPC_NONE, - BITFIELD(21, 1) /* index 880 */, - TILEGX_OPC_V1DDOTPU, TILEGX_OPC_NONE, - BITFIELD(21, 1) /* index 883 */, - TILEGX_OPC_V1DOTPUA, TILEGX_OPC_NONE, - BITFIELD(21, 1) /* index 886 */, - TILEGX_OPC_V1DOTPU, TILEGX_OPC_NONE, - BITFIELD(18, 4) /* index 889 */, - TILEGX_OPC_NONE, TILEGX_OPC_ROTLI, TILEGX_OPC_SHLI, TILEGX_OPC_SHLXI, - TILEGX_OPC_SHRSI, TILEGX_OPC_SHRUI, TILEGX_OPC_SHRUXI, TILEGX_OPC_V1SHLI, - TILEGX_OPC_V1SHRSI, TILEGX_OPC_V1SHRUI, TILEGX_OPC_V2SHLI, - TILEGX_OPC_V2SHRSI, TILEGX_OPC_V2SHRUI, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, - BITFIELD(0, 2) /* index 906 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(911), - BITFIELD(2, 2) /* index 911 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(916), - BITFIELD(4, 2) /* index 916 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(921), - BITFIELD(6, 2) /* index 921 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(926), - BITFIELD(8, 2) /* index 926 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(931), - BITFIELD(10, 2) /* index 931 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - TILEGX_OPC_INFOL, -}; - -static const unsigned short decode_X1_fsm[1266] = -{ - BITFIELD(53, 9) /* index 0 */, - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), CHILD(513), - CHILD(513), CHILD(513), CHILD(513), CHILD(513), TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, - TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_ADDXLI, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_BEQZT, - TILEGX_OPC_BEQZT, TILEGX_OPC_BEQZ, TILEGX_OPC_BEQZ, TILEGX_OPC_BGEZT, - TILEGX_OPC_BGEZT, TILEGX_OPC_BGEZ, TILEGX_OPC_BGEZ, TILEGX_OPC_BGTZT, - TILEGX_OPC_BGTZT, TILEGX_OPC_BGTZ, TILEGX_OPC_BGTZ, TILEGX_OPC_BLBCT, - TILEGX_OPC_BLBCT, TILEGX_OPC_BLBC, TILEGX_OPC_BLBC, TILEGX_OPC_BLBST, - TILEGX_OPC_BLBST, TILEGX_OPC_BLBS, TILEGX_OPC_BLBS, TILEGX_OPC_BLEZT, - TILEGX_OPC_BLEZT, TILEGX_OPC_BLEZ, TILEGX_OPC_BLEZ, TILEGX_OPC_BLTZT, - TILEGX_OPC_BLTZT, TILEGX_OPC_BLTZ, TILEGX_OPC_BLTZ, TILEGX_OPC_BNEZT, - TILEGX_OPC_BNEZT, TILEGX_OPC_BNEZ, TILEGX_OPC_BNEZ, CHILD(528), CHILD(578), - CHILD(598), CHILD(703), CHILD(723), CHILD(728), CHILD(753), CHILD(758), - CHILD(763), CHILD(768), CHILD(773), CHILD(778), TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, - TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_JAL, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, TILEGX_OPC_J, - CHILD(783), CHILD(800), CHILD(832), CHILD(849), CHILD(1168), CHILD(1185), - CHILD(1202), TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, CHILD(1219), TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), CHILD(1236), - CHILD(1236), - BITFIELD(37, 2) /* index 513 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, CHILD(518), - BITFIELD(39, 2) /* index 518 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, CHILD(523), - BITFIELD(41, 2) /* index 523 */, - TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_ADDLI, TILEGX_OPC_MOVELI, - BITFIELD(51, 2) /* index 528 */, - TILEGX_OPC_NONE, CHILD(533), TILEGX_OPC_ADDXI, CHILD(548), - BITFIELD(37, 2) /* index 533 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(538), - BITFIELD(39, 2) /* index 538 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(543), - BITFIELD(41, 2) /* index 543 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_MOVEI, - BITFIELD(31, 2) /* index 548 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(553), - BITFIELD(33, 2) /* index 553 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(558), - BITFIELD(35, 2) /* index 558 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(563), - BITFIELD(37, 2) /* index 563 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(568), - BITFIELD(39, 2) /* index 568 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(573), - BITFIELD(41, 2) /* index 573 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_INFO, - BITFIELD(51, 2) /* index 578 */, - TILEGX_OPC_CMPEQI, TILEGX_OPC_CMPLTSI, TILEGX_OPC_CMPLTUI, CHILD(583), - BITFIELD(31, 2) /* index 583 */, - TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, CHILD(588), - BITFIELD(33, 2) /* index 588 */, - TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, CHILD(593), - BITFIELD(35, 2) /* index 593 */, - TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, TILEGX_OPC_LD1S_ADD, - TILEGX_OPC_PREFETCH_ADD_L1_FAULT, - BITFIELD(51, 2) /* index 598 */, - CHILD(603), CHILD(618), CHILD(633), CHILD(648), - BITFIELD(31, 2) /* index 603 */, - TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, CHILD(608), - BITFIELD(33, 2) /* index 608 */, - TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, CHILD(613), - BITFIELD(35, 2) /* index 613 */, - TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, TILEGX_OPC_LD1U_ADD, - TILEGX_OPC_PREFETCH_ADD_L1, - BITFIELD(31, 2) /* index 618 */, - TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, CHILD(623), - BITFIELD(33, 2) /* index 623 */, - TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, CHILD(628), - BITFIELD(35, 2) /* index 628 */, - TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, TILEGX_OPC_LD2S_ADD, - TILEGX_OPC_PREFETCH_ADD_L2_FAULT, - BITFIELD(31, 2) /* index 633 */, - TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, CHILD(638), - BITFIELD(33, 2) /* index 638 */, - TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, CHILD(643), - BITFIELD(35, 2) /* index 643 */, - TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, TILEGX_OPC_LD2U_ADD, - TILEGX_OPC_PREFETCH_ADD_L2, - BITFIELD(31, 2) /* index 648 */, - CHILD(653), CHILD(653), CHILD(653), CHILD(673), - BITFIELD(43, 2) /* index 653 */, - CHILD(658), TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, - BITFIELD(45, 2) /* index 658 */, - CHILD(663), TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, - BITFIELD(47, 2) /* index 663 */, - CHILD(668), TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, - BITFIELD(49, 2) /* index 668 */, - TILEGX_OPC_LD4S_TLS, TILEGX_OPC_LD4S_ADD, TILEGX_OPC_LD4S_ADD, - TILEGX_OPC_LD4S_ADD, - BITFIELD(33, 2) /* index 673 */, - CHILD(653), CHILD(653), CHILD(653), CHILD(678), - BITFIELD(35, 2) /* index 678 */, - CHILD(653), CHILD(653), CHILD(653), CHILD(683), - BITFIELD(43, 2) /* index 683 */, - CHILD(688), TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - TILEGX_OPC_PREFETCH_ADD_L3_FAULT, TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - BITFIELD(45, 2) /* index 688 */, - CHILD(693), TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - TILEGX_OPC_PREFETCH_ADD_L3_FAULT, TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - BITFIELD(47, 2) /* index 693 */, - CHILD(698), TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - TILEGX_OPC_PREFETCH_ADD_L3_FAULT, TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - BITFIELD(49, 2) /* index 698 */, - TILEGX_OPC_LD4S_TLS, TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - TILEGX_OPC_PREFETCH_ADD_L3_FAULT, TILEGX_OPC_PREFETCH_ADD_L3_FAULT, - BITFIELD(51, 2) /* index 703 */, - CHILD(708), TILEGX_OPC_LDNT1S_ADD, TILEGX_OPC_LDNT1U_ADD, - TILEGX_OPC_LDNT2S_ADD, - BITFIELD(31, 2) /* index 708 */, - TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, CHILD(713), - BITFIELD(33, 2) /* index 713 */, - TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, CHILD(718), - BITFIELD(35, 2) /* index 718 */, - TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, TILEGX_OPC_LD4U_ADD, - TILEGX_OPC_PREFETCH_ADD_L3, - BITFIELD(51, 2) /* index 723 */, - TILEGX_OPC_LDNT2U_ADD, TILEGX_OPC_LDNT4S_ADD, TILEGX_OPC_LDNT4U_ADD, - TILEGX_OPC_LDNT_ADD, - BITFIELD(51, 2) /* index 728 */, - CHILD(733), TILEGX_OPC_LDNA_ADD, TILEGX_OPC_MFSPR, TILEGX_OPC_MTSPR, - BITFIELD(43, 2) /* index 733 */, - CHILD(738), TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, - BITFIELD(45, 2) /* index 738 */, - CHILD(743), TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, - BITFIELD(47, 2) /* index 743 */, - CHILD(748), TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, - BITFIELD(49, 2) /* index 748 */, - TILEGX_OPC_LD_TLS, TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, TILEGX_OPC_LD_ADD, - BITFIELD(51, 2) /* index 753 */, - TILEGX_OPC_ORI, TILEGX_OPC_ST1_ADD, TILEGX_OPC_ST2_ADD, TILEGX_OPC_ST4_ADD, - BITFIELD(51, 2) /* index 758 */, - TILEGX_OPC_STNT1_ADD, TILEGX_OPC_STNT2_ADD, TILEGX_OPC_STNT4_ADD, - TILEGX_OPC_STNT_ADD, - BITFIELD(51, 2) /* index 763 */, - TILEGX_OPC_ST_ADD, TILEGX_OPC_V1ADDI, TILEGX_OPC_V1CMPEQI, - TILEGX_OPC_V1CMPLTSI, - BITFIELD(51, 2) /* index 768 */, - TILEGX_OPC_V1CMPLTUI, TILEGX_OPC_V1MAXUI, TILEGX_OPC_V1MINUI, - TILEGX_OPC_V2ADDI, - BITFIELD(51, 2) /* index 773 */, - TILEGX_OPC_V2CMPEQI, TILEGX_OPC_V2CMPLTSI, TILEGX_OPC_V2CMPLTUI, - TILEGX_OPC_V2MAXSI, - BITFIELD(51, 2) /* index 778 */, - TILEGX_OPC_V2MINSI, TILEGX_OPC_XORI, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(49, 4) /* index 783 */, - TILEGX_OPC_NONE, TILEGX_OPC_ADDXSC, TILEGX_OPC_ADDX, TILEGX_OPC_ADD, - TILEGX_OPC_AND, TILEGX_OPC_CMPEQ, TILEGX_OPC_CMPEXCH4, TILEGX_OPC_CMPEXCH, - TILEGX_OPC_CMPLES, TILEGX_OPC_CMPLEU, TILEGX_OPC_CMPLTS, TILEGX_OPC_CMPLTU, - TILEGX_OPC_CMPNE, TILEGX_OPC_DBLALIGN2, TILEGX_OPC_DBLALIGN4, - TILEGX_OPC_DBLALIGN6, - BITFIELD(49, 4) /* index 800 */, - TILEGX_OPC_EXCH4, TILEGX_OPC_EXCH, TILEGX_OPC_FETCHADD4, - TILEGX_OPC_FETCHADDGEZ4, TILEGX_OPC_FETCHADDGEZ, TILEGX_OPC_FETCHADD, - TILEGX_OPC_FETCHAND4, TILEGX_OPC_FETCHAND, TILEGX_OPC_FETCHOR4, - TILEGX_OPC_FETCHOR, TILEGX_OPC_MNZ, TILEGX_OPC_MZ, TILEGX_OPC_NOR, - CHILD(817), TILEGX_OPC_ROTL, TILEGX_OPC_SHL1ADDX, - BITFIELD(43, 2) /* index 817 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(822), - BITFIELD(45, 2) /* index 822 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(827), - BITFIELD(47, 2) /* index 827 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_MOVE, - BITFIELD(49, 4) /* index 832 */, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL2ADDX, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL3ADDX, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHLX, TILEGX_OPC_SHL, - TILEGX_OPC_SHRS, TILEGX_OPC_SHRUX, TILEGX_OPC_SHRU, TILEGX_OPC_ST1, - TILEGX_OPC_ST2, TILEGX_OPC_ST4, TILEGX_OPC_STNT1, TILEGX_OPC_STNT2, - TILEGX_OPC_STNT4, - BITFIELD(46, 7) /* index 849 */, - TILEGX_OPC_STNT, TILEGX_OPC_STNT, TILEGX_OPC_STNT, TILEGX_OPC_STNT, - TILEGX_OPC_STNT, TILEGX_OPC_STNT, TILEGX_OPC_STNT, TILEGX_OPC_STNT, - TILEGX_OPC_ST, TILEGX_OPC_ST, TILEGX_OPC_ST, TILEGX_OPC_ST, TILEGX_OPC_ST, - TILEGX_OPC_ST, TILEGX_OPC_ST, TILEGX_OPC_ST, TILEGX_OPC_SUBXSC, - TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBXSC, - TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBXSC, TILEGX_OPC_SUBX, - TILEGX_OPC_SUBX, TILEGX_OPC_SUBX, TILEGX_OPC_SUBX, TILEGX_OPC_SUBX, - TILEGX_OPC_SUBX, TILEGX_OPC_SUBX, TILEGX_OPC_SUBX, TILEGX_OPC_SUB, - TILEGX_OPC_SUB, TILEGX_OPC_SUB, TILEGX_OPC_SUB, TILEGX_OPC_SUB, - TILEGX_OPC_SUB, TILEGX_OPC_SUB, TILEGX_OPC_SUB, CHILD(978), CHILD(987), - CHILD(1066), CHILD(1150), CHILD(1159), TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADDUC, - TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADDUC, - TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADDUC, TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADD, - TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADD, - TILEGX_OPC_V1ADD, TILEGX_OPC_V1ADD, TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPEQ, - TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPEQ, - TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPEQ, TILEGX_OPC_V1CMPEQ, - TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLES, - TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLES, - TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLES, TILEGX_OPC_V1CMPLEU, - TILEGX_OPC_V1CMPLEU, TILEGX_OPC_V1CMPLEU, TILEGX_OPC_V1CMPLEU, - TILEGX_OPC_V1CMPLEU, TILEGX_OPC_V1CMPLEU, TILEGX_OPC_V1CMPLEU, - TILEGX_OPC_V1CMPLEU, TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTS, - TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTS, - TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTS, TILEGX_OPC_V1CMPLTS, - TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPLTU, - TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPLTU, - TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPLTU, TILEGX_OPC_V1CMPNE, - TILEGX_OPC_V1CMPNE, TILEGX_OPC_V1CMPNE, TILEGX_OPC_V1CMPNE, - TILEGX_OPC_V1CMPNE, TILEGX_OPC_V1CMPNE, TILEGX_OPC_V1CMPNE, - TILEGX_OPC_V1CMPNE, TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_H, - TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_H, - TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_H, TILEGX_OPC_V1INT_H, - TILEGX_OPC_V1INT_L, TILEGX_OPC_V1INT_L, TILEGX_OPC_V1INT_L, - TILEGX_OPC_V1INT_L, TILEGX_OPC_V1INT_L, TILEGX_OPC_V1INT_L, - TILEGX_OPC_V1INT_L, TILEGX_OPC_V1INT_L, - BITFIELD(43, 3) /* index 978 */, - TILEGX_OPC_NONE, TILEGX_OPC_DRAIN, TILEGX_OPC_DTLBPR, TILEGX_OPC_FINV, - TILEGX_OPC_FLUSHWB, TILEGX_OPC_FLUSH, TILEGX_OPC_FNOP, TILEGX_OPC_ICOH, - BITFIELD(43, 3) /* index 987 */, - CHILD(996), TILEGX_OPC_INV, TILEGX_OPC_IRET, TILEGX_OPC_JALRP, - TILEGX_OPC_JALR, TILEGX_OPC_JRP, TILEGX_OPC_JR, CHILD(1051), - BITFIELD(31, 2) /* index 996 */, - CHILD(1001), CHILD(1026), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(33, 2) /* index 1001 */, - TILEGX_OPC_ILL, TILEGX_OPC_ILL, TILEGX_OPC_ILL, CHILD(1006), - BITFIELD(35, 2) /* index 1006 */, - TILEGX_OPC_ILL, CHILD(1011), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(37, 2) /* index 1011 */, - TILEGX_OPC_ILL, CHILD(1016), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(39, 2) /* index 1016 */, - TILEGX_OPC_ILL, CHILD(1021), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(41, 2) /* index 1021 */, - TILEGX_OPC_ILL, TILEGX_OPC_ILL, TILEGX_OPC_BPT, TILEGX_OPC_ILL, - BITFIELD(33, 2) /* index 1026 */, - TILEGX_OPC_ILL, TILEGX_OPC_ILL, TILEGX_OPC_ILL, CHILD(1031), - BITFIELD(35, 2) /* index 1031 */, - TILEGX_OPC_ILL, CHILD(1036), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(37, 2) /* index 1036 */, - TILEGX_OPC_ILL, CHILD(1041), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(39, 2) /* index 1041 */, - TILEGX_OPC_ILL, CHILD(1046), TILEGX_OPC_ILL, TILEGX_OPC_ILL, - BITFIELD(41, 2) /* index 1046 */, - TILEGX_OPC_ILL, TILEGX_OPC_ILL, TILEGX_OPC_RAISE, TILEGX_OPC_ILL, - BITFIELD(31, 2) /* index 1051 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, CHILD(1056), - BITFIELD(33, 2) /* index 1056 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, CHILD(1061), - BITFIELD(35, 2) /* index 1061 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, - TILEGX_OPC_PREFETCH_L1_FAULT, - BITFIELD(43, 3) /* index 1066 */, - CHILD(1075), CHILD(1090), CHILD(1105), CHILD(1120), CHILD(1135), - TILEGX_OPC_LDNA, TILEGX_OPC_LDNT1S, TILEGX_OPC_LDNT1U, - BITFIELD(31, 2) /* index 1075 */, - TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, CHILD(1080), - BITFIELD(33, 2) /* index 1080 */, - TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, CHILD(1085), - BITFIELD(35, 2) /* index 1085 */, - TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_PREFETCH, - BITFIELD(31, 2) /* index 1090 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, CHILD(1095), - BITFIELD(33, 2) /* index 1095 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, CHILD(1100), - BITFIELD(35, 2) /* index 1100 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, - TILEGX_OPC_PREFETCH_L2_FAULT, - BITFIELD(31, 2) /* index 1105 */, - TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, CHILD(1110), - BITFIELD(33, 2) /* index 1110 */, - TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, CHILD(1115), - BITFIELD(35, 2) /* index 1115 */, - TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_PREFETCH_L2, - BITFIELD(31, 2) /* index 1120 */, - TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, CHILD(1125), - BITFIELD(33, 2) /* index 1125 */, - TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, CHILD(1130), - BITFIELD(35, 2) /* index 1130 */, - TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, - TILEGX_OPC_PREFETCH_L3_FAULT, - BITFIELD(31, 2) /* index 1135 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, CHILD(1140), - BITFIELD(33, 2) /* index 1140 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, CHILD(1145), - BITFIELD(35, 2) /* index 1145 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_PREFETCH_L3, - BITFIELD(43, 3) /* index 1150 */, - TILEGX_OPC_LDNT2S, TILEGX_OPC_LDNT2U, TILEGX_OPC_LDNT4S, TILEGX_OPC_LDNT4U, - TILEGX_OPC_LDNT, TILEGX_OPC_LD, TILEGX_OPC_LNK, TILEGX_OPC_MF, - BITFIELD(43, 3) /* index 1159 */, - TILEGX_OPC_NAP, TILEGX_OPC_NOP, TILEGX_OPC_SWINT0, TILEGX_OPC_SWINT1, - TILEGX_OPC_SWINT2, TILEGX_OPC_SWINT3, TILEGX_OPC_WH64, TILEGX_OPC_NONE, - BITFIELD(49, 4) /* index 1168 */, - TILEGX_OPC_V1MAXU, TILEGX_OPC_V1MINU, TILEGX_OPC_V1MNZ, TILEGX_OPC_V1MZ, - TILEGX_OPC_V1SHL, TILEGX_OPC_V1SHRS, TILEGX_OPC_V1SHRU, TILEGX_OPC_V1SUBUC, - TILEGX_OPC_V1SUB, TILEGX_OPC_V2ADDSC, TILEGX_OPC_V2ADD, TILEGX_OPC_V2CMPEQ, - TILEGX_OPC_V2CMPLES, TILEGX_OPC_V2CMPLEU, TILEGX_OPC_V2CMPLTS, - TILEGX_OPC_V2CMPLTU, - BITFIELD(49, 4) /* index 1185 */, - TILEGX_OPC_V2CMPNE, TILEGX_OPC_V2INT_H, TILEGX_OPC_V2INT_L, - TILEGX_OPC_V2MAXS, TILEGX_OPC_V2MINS, TILEGX_OPC_V2MNZ, TILEGX_OPC_V2MZ, - TILEGX_OPC_V2PACKH, TILEGX_OPC_V2PACKL, TILEGX_OPC_V2PACKUC, - TILEGX_OPC_V2SHLSC, TILEGX_OPC_V2SHL, TILEGX_OPC_V2SHRS, TILEGX_OPC_V2SHRU, - TILEGX_OPC_V2SUBSC, TILEGX_OPC_V2SUB, - BITFIELD(49, 4) /* index 1202 */, - TILEGX_OPC_V4ADDSC, TILEGX_OPC_V4ADD, TILEGX_OPC_V4INT_H, - TILEGX_OPC_V4INT_L, TILEGX_OPC_V4PACKSC, TILEGX_OPC_V4SHLSC, - TILEGX_OPC_V4SHL, TILEGX_OPC_V4SHRS, TILEGX_OPC_V4SHRU, TILEGX_OPC_V4SUBSC, - TILEGX_OPC_V4SUB, TILEGX_OPC_XOR, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(49, 4) /* index 1219 */, - TILEGX_OPC_NONE, TILEGX_OPC_ROTLI, TILEGX_OPC_SHLI, TILEGX_OPC_SHLXI, - TILEGX_OPC_SHRSI, TILEGX_OPC_SHRUI, TILEGX_OPC_SHRUXI, TILEGX_OPC_V1SHLI, - TILEGX_OPC_V1SHRSI, TILEGX_OPC_V1SHRUI, TILEGX_OPC_V2SHLI, - TILEGX_OPC_V2SHRSI, TILEGX_OPC_V2SHRUI, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, - BITFIELD(31, 2) /* index 1236 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(1241), - BITFIELD(33, 2) /* index 1241 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(1246), - BITFIELD(35, 2) /* index 1246 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(1251), - BITFIELD(37, 2) /* index 1251 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(1256), - BITFIELD(39, 2) /* index 1256 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - CHILD(1261), - BITFIELD(41, 2) /* index 1261 */, - TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, TILEGX_OPC_SHL16INSLI, - TILEGX_OPC_INFOL, -}; - -static const unsigned short decode_Y0_fsm[178] = -{ - BITFIELD(27, 4) /* index 0 */, - CHILD(17), TILEGX_OPC_ADDXI, CHILD(32), TILEGX_OPC_CMPEQI, - TILEGX_OPC_CMPLTSI, CHILD(62), CHILD(67), CHILD(118), CHILD(123), - CHILD(128), CHILD(133), CHILD(153), CHILD(158), CHILD(163), CHILD(168), - CHILD(173), - BITFIELD(6, 2) /* index 17 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(22), - BITFIELD(8, 2) /* index 22 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(27), - BITFIELD(10, 2) /* index 27 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_MOVEI, - BITFIELD(0, 2) /* index 32 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(37), - BITFIELD(2, 2) /* index 37 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(42), - BITFIELD(4, 2) /* index 42 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(47), - BITFIELD(6, 2) /* index 47 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(52), - BITFIELD(8, 2) /* index 52 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(57), - BITFIELD(10, 2) /* index 57 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_INFO, - BITFIELD(18, 2) /* index 62 */, - TILEGX_OPC_ADDX, TILEGX_OPC_ADD, TILEGX_OPC_SUBX, TILEGX_OPC_SUB, - BITFIELD(15, 5) /* index 67 */, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, - TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, - TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, CHILD(100), - CHILD(109), TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(12, 3) /* index 100 */, - TILEGX_OPC_NONE, TILEGX_OPC_CLZ, TILEGX_OPC_CTZ, TILEGX_OPC_FNOP, - TILEGX_OPC_FSINGLE_PACK1, TILEGX_OPC_NOP, TILEGX_OPC_PCNT, - TILEGX_OPC_REVBITS, - BITFIELD(12, 3) /* index 109 */, - TILEGX_OPC_REVBYTES, TILEGX_OPC_TBLIDXB0, TILEGX_OPC_TBLIDXB1, - TILEGX_OPC_TBLIDXB2, TILEGX_OPC_TBLIDXB3, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - TILEGX_OPC_NONE, - BITFIELD(18, 2) /* index 118 */, - TILEGX_OPC_CMPLES, TILEGX_OPC_CMPLEU, TILEGX_OPC_CMPLTS, TILEGX_OPC_CMPLTU, - BITFIELD(18, 2) /* index 123 */, - TILEGX_OPC_CMPEQ, TILEGX_OPC_CMPNE, TILEGX_OPC_MULAX, TILEGX_OPC_MULX, - BITFIELD(18, 2) /* index 128 */, - TILEGX_OPC_CMOVEQZ, TILEGX_OPC_CMOVNEZ, TILEGX_OPC_MNZ, TILEGX_OPC_MZ, - BITFIELD(18, 2) /* index 133 */, - TILEGX_OPC_AND, TILEGX_OPC_NOR, CHILD(138), TILEGX_OPC_XOR, - BITFIELD(12, 2) /* index 138 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(143), - BITFIELD(14, 2) /* index 143 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(148), - BITFIELD(16, 2) /* index 148 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_MOVE, - BITFIELD(18, 2) /* index 153 */, - TILEGX_OPC_ROTL, TILEGX_OPC_SHL, TILEGX_OPC_SHRS, TILEGX_OPC_SHRU, - BITFIELD(18, 2) /* index 158 */, - TILEGX_OPC_NONE, TILEGX_OPC_SHL1ADDX, TILEGX_OPC_SHL2ADDX, - TILEGX_OPC_SHL3ADDX, - BITFIELD(18, 2) /* index 163 */, - TILEGX_OPC_MUL_HS_HS, TILEGX_OPC_MUL_HU_HU, TILEGX_OPC_MUL_LS_LS, - TILEGX_OPC_MUL_LU_LU, - BITFIELD(18, 2) /* index 168 */, - TILEGX_OPC_MULA_HS_HS, TILEGX_OPC_MULA_HU_HU, TILEGX_OPC_MULA_LS_LS, - TILEGX_OPC_MULA_LU_LU, - BITFIELD(18, 2) /* index 173 */, - TILEGX_OPC_ROTLI, TILEGX_OPC_SHLI, TILEGX_OPC_SHRSI, TILEGX_OPC_SHRUI, -}; - -static const unsigned short decode_Y1_fsm[167] = -{ - BITFIELD(58, 4) /* index 0 */, - TILEGX_OPC_NONE, CHILD(17), TILEGX_OPC_ADDXI, CHILD(32), TILEGX_OPC_CMPEQI, - TILEGX_OPC_CMPLTSI, CHILD(62), CHILD(67), CHILD(117), CHILD(122), - CHILD(127), CHILD(132), CHILD(152), CHILD(157), CHILD(162), TILEGX_OPC_NONE, - BITFIELD(37, 2) /* index 17 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(22), - BITFIELD(39, 2) /* index 22 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, CHILD(27), - BITFIELD(41, 2) /* index 27 */, - TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_ADDI, TILEGX_OPC_MOVEI, - BITFIELD(31, 2) /* index 32 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(37), - BITFIELD(33, 2) /* index 37 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(42), - BITFIELD(35, 2) /* index 42 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(47), - BITFIELD(37, 2) /* index 47 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(52), - BITFIELD(39, 2) /* index 52 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, CHILD(57), - BITFIELD(41, 2) /* index 57 */, - TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_ANDI, TILEGX_OPC_INFO, - BITFIELD(49, 2) /* index 62 */, - TILEGX_OPC_ADDX, TILEGX_OPC_ADD, TILEGX_OPC_SUBX, TILEGX_OPC_SUB, - BITFIELD(47, 4) /* index 67 */, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL1ADD, - TILEGX_OPC_SHL1ADD, TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, - TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL2ADD, TILEGX_OPC_SHL3ADD, - TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, TILEGX_OPC_SHL3ADD, CHILD(84), - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_NONE, - BITFIELD(43, 3) /* index 84 */, - CHILD(93), CHILD(96), CHILD(99), CHILD(102), CHILD(105), CHILD(108), - CHILD(111), CHILD(114), - BITFIELD(46, 1) /* index 93 */, - TILEGX_OPC_NONE, TILEGX_OPC_FNOP, - BITFIELD(46, 1) /* index 96 */, - TILEGX_OPC_NONE, TILEGX_OPC_ILL, - BITFIELD(46, 1) /* index 99 */, - TILEGX_OPC_NONE, TILEGX_OPC_JALRP, - BITFIELD(46, 1) /* index 102 */, - TILEGX_OPC_NONE, TILEGX_OPC_JALR, - BITFIELD(46, 1) /* index 105 */, - TILEGX_OPC_NONE, TILEGX_OPC_JRP, - BITFIELD(46, 1) /* index 108 */, - TILEGX_OPC_NONE, TILEGX_OPC_JR, - BITFIELD(46, 1) /* index 111 */, - TILEGX_OPC_NONE, TILEGX_OPC_LNK, - BITFIELD(46, 1) /* index 114 */, - TILEGX_OPC_NONE, TILEGX_OPC_NOP, - BITFIELD(49, 2) /* index 117 */, - TILEGX_OPC_CMPLES, TILEGX_OPC_CMPLEU, TILEGX_OPC_CMPLTS, TILEGX_OPC_CMPLTU, - BITFIELD(49, 2) /* index 122 */, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_CMPEQ, TILEGX_OPC_CMPNE, - BITFIELD(49, 2) /* index 127 */, - TILEGX_OPC_NONE, TILEGX_OPC_NONE, TILEGX_OPC_MNZ, TILEGX_OPC_MZ, - BITFIELD(49, 2) /* index 132 */, - TILEGX_OPC_AND, TILEGX_OPC_NOR, CHILD(137), TILEGX_OPC_XOR, - BITFIELD(43, 2) /* index 137 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(142), - BITFIELD(45, 2) /* index 142 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, CHILD(147), - BITFIELD(47, 2) /* index 147 */, - TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_OR, TILEGX_OPC_MOVE, - BITFIELD(49, 2) /* index 152 */, - TILEGX_OPC_ROTL, TILEGX_OPC_SHL, TILEGX_OPC_SHRS, TILEGX_OPC_SHRU, - BITFIELD(49, 2) /* index 157 */, - TILEGX_OPC_NONE, TILEGX_OPC_SHL1ADDX, TILEGX_OPC_SHL2ADDX, - TILEGX_OPC_SHL3ADDX, - BITFIELD(49, 2) /* index 162 */, - TILEGX_OPC_ROTLI, TILEGX_OPC_SHLI, TILEGX_OPC_SHRSI, TILEGX_OPC_SHRUI, -}; - -static const unsigned short decode_Y2_fsm[118] = -{ - BITFIELD(62, 2) /* index 0 */, - TILEGX_OPC_NONE, CHILD(5), CHILD(66), CHILD(109), - BITFIELD(55, 3) /* index 5 */, - CHILD(14), CHILD(14), CHILD(14), CHILD(17), CHILD(40), CHILD(40), CHILD(40), - CHILD(43), - BITFIELD(26, 1) /* index 14 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1U, - BITFIELD(26, 1) /* index 17 */, - CHILD(20), CHILD(30), - BITFIELD(51, 2) /* index 20 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, CHILD(25), - BITFIELD(53, 2) /* index 25 */, - TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, TILEGX_OPC_LD1S, - TILEGX_OPC_PREFETCH_L1_FAULT, - BITFIELD(51, 2) /* index 30 */, - TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, CHILD(35), - BITFIELD(53, 2) /* index 35 */, - TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_LD1U, TILEGX_OPC_PREFETCH, - BITFIELD(26, 1) /* index 40 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2U, - BITFIELD(26, 1) /* index 43 */, - CHILD(46), CHILD(56), - BITFIELD(51, 2) /* index 46 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, CHILD(51), - BITFIELD(53, 2) /* index 51 */, - TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, TILEGX_OPC_LD2S, - TILEGX_OPC_PREFETCH_L2_FAULT, - BITFIELD(51, 2) /* index 56 */, - TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, CHILD(61), - BITFIELD(53, 2) /* index 61 */, - TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_LD2U, TILEGX_OPC_PREFETCH_L2, - BITFIELD(56, 2) /* index 66 */, - CHILD(71), CHILD(74), CHILD(90), CHILD(93), - BITFIELD(26, 1) /* index 71 */, - TILEGX_OPC_NONE, TILEGX_OPC_LD4S, - BITFIELD(26, 1) /* index 74 */, - TILEGX_OPC_NONE, CHILD(77), - BITFIELD(51, 2) /* index 77 */, - TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, CHILD(82), - BITFIELD(53, 2) /* index 82 */, - TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, TILEGX_OPC_LD4S, CHILD(87), - BITFIELD(55, 1) /* index 87 */, - TILEGX_OPC_LD4S, TILEGX_OPC_PREFETCH_L3_FAULT, - BITFIELD(26, 1) /* index 90 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD, - BITFIELD(26, 1) /* index 93 */, - CHILD(96), TILEGX_OPC_LD, - BITFIELD(51, 2) /* index 96 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, CHILD(101), - BITFIELD(53, 2) /* index 101 */, - TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, TILEGX_OPC_LD4U, CHILD(106), - BITFIELD(55, 1) /* index 106 */, - TILEGX_OPC_LD4U, TILEGX_OPC_PREFETCH_L3, - BITFIELD(26, 1) /* index 109 */, - CHILD(112), CHILD(115), - BITFIELD(57, 1) /* index 112 */, - TILEGX_OPC_ST1, TILEGX_OPC_ST4, - BITFIELD(57, 1) /* index 115 */, - TILEGX_OPC_ST2, TILEGX_OPC_ST, -}; - -#undef BITFIELD -#undef CHILD - -const unsigned short * const -tilegx_bundle_decoder_fsms[TILEGX_NUM_PIPELINE_ENCODINGS] = -{ - decode_X0_fsm, - decode_X1_fsm, - decode_Y0_fsm, - decode_Y1_fsm, - decode_Y2_fsm -}; - -const struct tilegx_operand tilegx_operands[35] = -{ - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM8_X0), - 8, 1, 0, 0, 0, 0, - create_Imm8_X0, get_Imm8_X0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM8_X1), - 8, 1, 0, 0, 0, 0, - create_Imm8_X1, get_Imm8_X1 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM8_Y0), - 8, 1, 0, 0, 0, 0, - create_Imm8_Y0, get_Imm8_Y0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM8_Y1), - 8, 1, 0, 0, 0, 0, - create_Imm8_Y1, get_Imm8_Y1 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM16_X0_HW0_LAST), - 16, 1, 0, 0, 0, 0, - create_Imm16_X0, get_Imm16_X0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_IMM16_X1_HW0_LAST), - 16, 1, 0, 0, 0, 0, - create_Imm16_X1, get_Imm16_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 0, 1, 0, 0, - create_Dest_X1, get_Dest_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcA_X1, get_SrcA_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 0, 1, 0, 0, - create_Dest_X0, get_Dest_X0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcA_X0, get_SrcA_X0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 0, 1, 0, 0, - create_Dest_Y0, get_Dest_Y0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcA_Y0, get_SrcA_Y0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 0, 1, 0, 0, - create_Dest_Y1, get_Dest_Y1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcA_Y1, get_SrcA_Y1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcA_Y2, get_SrcA_Y2 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 1, 0, 0, - create_SrcA_X1, get_SrcA_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcB_X0, get_SrcB_X0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcB_X1, get_SrcB_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcB_Y0, get_SrcB_Y0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcB_Y1, get_SrcB_Y1 - }, - { - TILEGX_OP_TYPE_ADDRESS, BFD_RELOC(TILEGX_BROFF_X1), - 17, 1, 0, 0, 1, TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES, - create_BrOff_X1, get_BrOff_X1 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_MMSTART_X0), - 6, 0, 0, 0, 0, 0, - create_BFStart_X0, get_BFStart_X0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_MMEND_X0), - 6, 0, 0, 0, 0, 0, - create_BFEnd_X0, get_BFEnd_X0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 1, 0, 0, - create_Dest_X0, get_Dest_X0 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 1, 0, 0, - create_Dest_Y0, get_Dest_Y0 - }, - { - TILEGX_OP_TYPE_ADDRESS, BFD_RELOC(TILEGX_JUMPOFF_X1), - 27, 1, 0, 0, 1, TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES, - create_JumpOff_X1, get_JumpOff_X1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 0, 1, 0, 0, - create_SrcBDest_Y2, get_SrcBDest_Y2 - }, - { - TILEGX_OP_TYPE_SPR, BFD_RELOC(TILEGX_MF_IMM14_X1), - 14, 0, 0, 0, 0, 0, - create_MF_Imm14_X1, get_MF_Imm14_X1 - }, - { - TILEGX_OP_TYPE_SPR, BFD_RELOC(TILEGX_MT_IMM14_X1), - 14, 0, 0, 0, 0, 0, - create_MT_Imm14_X1, get_MT_Imm14_X1 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_SHAMT_X0), - 6, 0, 0, 0, 0, 0, - create_ShAmt_X0, get_ShAmt_X0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_SHAMT_X1), - 6, 0, 0, 0, 0, 0, - create_ShAmt_X1, get_ShAmt_X1 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_SHAMT_Y0), - 6, 0, 0, 0, 0, 0, - create_ShAmt_Y0, get_ShAmt_Y0 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_SHAMT_Y1), - 6, 0, 0, 0, 0, 0, - create_ShAmt_Y1, get_ShAmt_Y1 - }, - { - TILEGX_OP_TYPE_REGISTER, BFD_RELOC(NONE), - 6, 0, 1, 0, 0, 0, - create_SrcBDest_Y2, get_SrcBDest_Y2 - }, - { - TILEGX_OP_TYPE_IMMEDIATE, BFD_RELOC(TILEGX_DEST_IMM8_X1), - 8, 1, 0, 0, 0, 0, - create_Dest_Imm8_X1, get_Dest_Imm8_X1 - } -}; - -/* Given a set of bundle bits and a specific pipe, returns which - * instruction the bundle contains in that pipe. - */ -const struct tilegx_opcode * -find_opcode(tilegx_bundle_bits bits, tilegx_pipeline pipe) -{ - const unsigned short *table = tilegx_bundle_decoder_fsms[pipe]; - int index = 0; - - while (1) - { - unsigned short bitspec = table[index]; - unsigned int bitfield = - ((unsigned int)(bits >> (bitspec & 63))) & (bitspec >> 6); - - unsigned short next = table[index + 1 + bitfield]; - if (next <= TILEGX_OPC_NONE) - return &tilegx_opcodes[next]; - - index = next - TILEGX_OPC_NONE; - } -} - -int -parse_insn_tilegx(tilegx_bundle_bits bits, - unsigned long long pc, - struct tilegx_decoded_instruction - decoded[TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE]) -{ - int num_instructions = 0; - int pipe; - - int min_pipe, max_pipe; - if ((bits & TILEGX_BUNDLE_MODE_MASK) == 0) - { - min_pipe = TILEGX_PIPELINE_X0; - max_pipe = TILEGX_PIPELINE_X1; - } - else - { - min_pipe = TILEGX_PIPELINE_Y0; - max_pipe = TILEGX_PIPELINE_Y2; - } - - /* For each pipe, find an instruction that fits. */ - for (pipe = min_pipe; pipe <= max_pipe; pipe++) - { - const struct tilegx_opcode *opc; - struct tilegx_decoded_instruction *d; - int i; - - d = &decoded[num_instructions++]; - opc = find_opcode (bits, (tilegx_pipeline)pipe); - d->opcode = opc; - - /* Decode each operand, sign extending, etc. as appropriate. */ - for (i = 0; i < opc->num_operands; i++) - { - const struct tilegx_operand *op = - &tilegx_operands[opc->operands[pipe][i]]; - int raw_opval = op->extract (bits); - long long opval; - - if (op->is_signed) - { - /* Sign-extend the operand. */ - int shift = (int)((sizeof(int) * 8) - op->num_bits); - raw_opval = (raw_opval << shift) >> shift; - } - - /* Adjust PC-relative scaled branch offsets. */ - if (op->type == TILEGX_OP_TYPE_ADDRESS) - opval = (raw_opval * TILEGX_BUNDLE_SIZE_IN_BYTES) + pc; - else - opval = raw_opval; - - /* Record the final value. */ - d->operands[i] = op; - d->operand_values[i] = opval; - } - } - - return num_instructions; -} - -struct tilegx_spr -{ - /* The number */ - int number; - - /* The name */ - const char *name; -}; - -static int -tilegx_spr_compare (const void *a_ptr, const void *b_ptr) -{ - const struct tilegx_spr *a = (const struct tilegx_spr *) a_ptr; - const struct tilegx_spr *b = (const struct tilegx_spr *) b_ptr; - return (a->number - b->number); -} - -const struct tilegx_spr tilegx_sprs[] = { - { 0, "MPL_MEM_ERROR_SET_0" }, - { 1, "MPL_MEM_ERROR_SET_1" }, - { 2, "MPL_MEM_ERROR_SET_2" }, - { 3, "MPL_MEM_ERROR_SET_3" }, - { 4, "MPL_MEM_ERROR" }, - { 5, "MEM_ERROR_CBOX_ADDR" }, - { 6, "MEM_ERROR_CBOX_STATUS" }, - { 7, "MEM_ERROR_ENABLE" }, - { 8, "MEM_ERROR_MBOX_ADDR" }, - { 9, "MEM_ERROR_MBOX_STATUS" }, - { 10, "SBOX_ERROR" }, - { 11, "XDN_DEMUX_ERROR" }, - { 256, "MPL_SINGLE_STEP_3_SET_0" }, - { 257, "MPL_SINGLE_STEP_3_SET_1" }, - { 258, "MPL_SINGLE_STEP_3_SET_2" }, - { 259, "MPL_SINGLE_STEP_3_SET_3" }, - { 260, "MPL_SINGLE_STEP_3" }, - { 261, "SINGLE_STEP_CONTROL_3" }, - { 512, "MPL_SINGLE_STEP_2_SET_0" }, - { 513, "MPL_SINGLE_STEP_2_SET_1" }, - { 514, "MPL_SINGLE_STEP_2_SET_2" }, - { 515, "MPL_SINGLE_STEP_2_SET_3" }, - { 516, "MPL_SINGLE_STEP_2" }, - { 517, "SINGLE_STEP_CONTROL_2" }, - { 768, "MPL_SINGLE_STEP_1_SET_0" }, - { 769, "MPL_SINGLE_STEP_1_SET_1" }, - { 770, "MPL_SINGLE_STEP_1_SET_2" }, - { 771, "MPL_SINGLE_STEP_1_SET_3" }, - { 772, "MPL_SINGLE_STEP_1" }, - { 773, "SINGLE_STEP_CONTROL_1" }, - { 1024, "MPL_SINGLE_STEP_0_SET_0" }, - { 1025, "MPL_SINGLE_STEP_0_SET_1" }, - { 1026, "MPL_SINGLE_STEP_0_SET_2" }, - { 1027, "MPL_SINGLE_STEP_0_SET_3" }, - { 1028, "MPL_SINGLE_STEP_0" }, - { 1029, "SINGLE_STEP_CONTROL_0" }, - { 1280, "MPL_IDN_COMPLETE_SET_0" }, - { 1281, "MPL_IDN_COMPLETE_SET_1" }, - { 1282, "MPL_IDN_COMPLETE_SET_2" }, - { 1283, "MPL_IDN_COMPLETE_SET_3" }, - { 1284, "MPL_IDN_COMPLETE" }, - { 1285, "IDN_COMPLETE_PENDING" }, - { 1536, "MPL_UDN_COMPLETE_SET_0" }, - { 1537, "MPL_UDN_COMPLETE_SET_1" }, - { 1538, "MPL_UDN_COMPLETE_SET_2" }, - { 1539, "MPL_UDN_COMPLETE_SET_3" }, - { 1540, "MPL_UDN_COMPLETE" }, - { 1541, "UDN_COMPLETE_PENDING" }, - { 1792, "MPL_ITLB_MISS_SET_0" }, - { 1793, "MPL_ITLB_MISS_SET_1" }, - { 1794, "MPL_ITLB_MISS_SET_2" }, - { 1795, "MPL_ITLB_MISS_SET_3" }, - { 1796, "MPL_ITLB_MISS" }, - { 1797, "ITLB_TSB_BASE_ADDR_0" }, - { 1798, "ITLB_TSB_BASE_ADDR_1" }, - { 1920, "ITLB_CURRENT_ATTR" }, - { 1921, "ITLB_CURRENT_PA" }, - { 1922, "ITLB_CURRENT_VA" }, - { 1923, "ITLB_INDEX" }, - { 1924, "ITLB_MATCH_0" }, - { 1925, "ITLB_PERF" }, - { 1926, "ITLB_PR" }, - { 1927, "ITLB_TSB_ADDR_0" }, - { 1928, "ITLB_TSB_ADDR_1" }, - { 1929, "ITLB_TSB_FILL_CURRENT_ATTR" }, - { 1930, "ITLB_TSB_FILL_MATCH" }, - { 1931, "NUMBER_ITLB" }, - { 1932, "REPLACEMENT_ITLB" }, - { 1933, "WIRED_ITLB" }, - { 2048, "MPL_ILL_SET_0" }, - { 2049, "MPL_ILL_SET_1" }, - { 2050, "MPL_ILL_SET_2" }, - { 2051, "MPL_ILL_SET_3" }, - { 2052, "MPL_ILL" }, - { 2304, "MPL_GPV_SET_0" }, - { 2305, "MPL_GPV_SET_1" }, - { 2306, "MPL_GPV_SET_2" }, - { 2307, "MPL_GPV_SET_3" }, - { 2308, "MPL_GPV" }, - { 2309, "GPV_REASON" }, - { 2560, "MPL_IDN_ACCESS_SET_0" }, - { 2561, "MPL_IDN_ACCESS_SET_1" }, - { 2562, "MPL_IDN_ACCESS_SET_2" }, - { 2563, "MPL_IDN_ACCESS_SET_3" }, - { 2564, "MPL_IDN_ACCESS" }, - { 2565, "IDN_DEMUX_COUNT_0" }, - { 2566, "IDN_DEMUX_COUNT_1" }, - { 2567, "IDN_FLUSH_EGRESS" }, - { 2568, "IDN_PENDING" }, - { 2569, "IDN_ROUTE_ORDER" }, - { 2570, "IDN_SP_FIFO_CNT" }, - { 2688, "IDN_DATA_AVAIL" }, - { 2816, "MPL_UDN_ACCESS_SET_0" }, - { 2817, "MPL_UDN_ACCESS_SET_1" }, - { 2818, "MPL_UDN_ACCESS_SET_2" }, - { 2819, "MPL_UDN_ACCESS_SET_3" }, - { 2820, "MPL_UDN_ACCESS" }, - { 2821, "UDN_DEMUX_COUNT_0" }, - { 2822, "UDN_DEMUX_COUNT_1" }, - { 2823, "UDN_DEMUX_COUNT_2" }, - { 2824, "UDN_DEMUX_COUNT_3" }, - { 2825, "UDN_FLUSH_EGRESS" }, - { 2826, "UDN_PENDING" }, - { 2827, "UDN_ROUTE_ORDER" }, - { 2828, "UDN_SP_FIFO_CNT" }, - { 2944, "UDN_DATA_AVAIL" }, - { 3072, "MPL_SWINT_3_SET_0" }, - { 3073, "MPL_SWINT_3_SET_1" }, - { 3074, "MPL_SWINT_3_SET_2" }, - { 3075, "MPL_SWINT_3_SET_3" }, - { 3076, "MPL_SWINT_3" }, - { 3328, "MPL_SWINT_2_SET_0" }, - { 3329, "MPL_SWINT_2_SET_1" }, - { 3330, "MPL_SWINT_2_SET_2" }, - { 3331, "MPL_SWINT_2_SET_3" }, - { 3332, "MPL_SWINT_2" }, - { 3584, "MPL_SWINT_1_SET_0" }, - { 3585, "MPL_SWINT_1_SET_1" }, - { 3586, "MPL_SWINT_1_SET_2" }, - { 3587, "MPL_SWINT_1_SET_3" }, - { 3588, "MPL_SWINT_1" }, - { 3840, "MPL_SWINT_0_SET_0" }, - { 3841, "MPL_SWINT_0_SET_1" }, - { 3842, "MPL_SWINT_0_SET_2" }, - { 3843, "MPL_SWINT_0_SET_3" }, - { 3844, "MPL_SWINT_0" }, - { 4096, "MPL_ILL_TRANS_SET_0" }, - { 4097, "MPL_ILL_TRANS_SET_1" }, - { 4098, "MPL_ILL_TRANS_SET_2" }, - { 4099, "MPL_ILL_TRANS_SET_3" }, - { 4100, "MPL_ILL_TRANS" }, - { 4101, "ILL_TRANS_REASON" }, - { 4102, "ILL_VA_PC" }, - { 4352, "MPL_UNALIGN_DATA_SET_0" }, - { 4353, "MPL_UNALIGN_DATA_SET_1" }, - { 4354, "MPL_UNALIGN_DATA_SET_2" }, - { 4355, "MPL_UNALIGN_DATA_SET_3" }, - { 4356, "MPL_UNALIGN_DATA" }, - { 4608, "MPL_DTLB_MISS_SET_0" }, - { 4609, "MPL_DTLB_MISS_SET_1" }, - { 4610, "MPL_DTLB_MISS_SET_2" }, - { 4611, "MPL_DTLB_MISS_SET_3" }, - { 4612, "MPL_DTLB_MISS" }, - { 4613, "DTLB_TSB_BASE_ADDR_0" }, - { 4614, "DTLB_TSB_BASE_ADDR_1" }, - { 4736, "AAR" }, - { 4737, "CACHE_PINNED_WAYS" }, - { 4738, "DTLB_BAD_ADDR" }, - { 4739, "DTLB_BAD_ADDR_REASON" }, - { 4740, "DTLB_CURRENT_ATTR" }, - { 4741, "DTLB_CURRENT_PA" }, - { 4742, "DTLB_CURRENT_VA" }, - { 4743, "DTLB_INDEX" }, - { 4744, "DTLB_MATCH_0" }, - { 4745, "DTLB_PERF" }, - { 4746, "DTLB_TSB_ADDR_0" }, - { 4747, "DTLB_TSB_ADDR_1" }, - { 4748, "DTLB_TSB_FILL_CURRENT_ATTR" }, - { 4749, "DTLB_TSB_FILL_MATCH" }, - { 4750, "NUMBER_DTLB" }, - { 4751, "REPLACEMENT_DTLB" }, - { 4752, "WIRED_DTLB" }, - { 4864, "MPL_DTLB_ACCESS_SET_0" }, - { 4865, "MPL_DTLB_ACCESS_SET_1" }, - { 4866, "MPL_DTLB_ACCESS_SET_2" }, - { 4867, "MPL_DTLB_ACCESS_SET_3" }, - { 4868, "MPL_DTLB_ACCESS" }, - { 5120, "MPL_IDN_FIREWALL_SET_0" }, - { 5121, "MPL_IDN_FIREWALL_SET_1" }, - { 5122, "MPL_IDN_FIREWALL_SET_2" }, - { 5123, "MPL_IDN_FIREWALL_SET_3" }, - { 5124, "MPL_IDN_FIREWALL" }, - { 5125, "IDN_DIRECTION_PROTECT" }, - { 5376, "MPL_UDN_FIREWALL_SET_0" }, - { 5377, "MPL_UDN_FIREWALL_SET_1" }, - { 5378, "MPL_UDN_FIREWALL_SET_2" }, - { 5379, "MPL_UDN_FIREWALL_SET_3" }, - { 5380, "MPL_UDN_FIREWALL" }, - { 5381, "UDN_DIRECTION_PROTECT" }, - { 5632, "MPL_TILE_TIMER_SET_0" }, - { 5633, "MPL_TILE_TIMER_SET_1" }, - { 5634, "MPL_TILE_TIMER_SET_2" }, - { 5635, "MPL_TILE_TIMER_SET_3" }, - { 5636, "MPL_TILE_TIMER" }, - { 5637, "TILE_TIMER_CONTROL" }, - { 5888, "MPL_AUX_TILE_TIMER_SET_0" }, - { 5889, "MPL_AUX_TILE_TIMER_SET_1" }, - { 5890, "MPL_AUX_TILE_TIMER_SET_2" }, - { 5891, "MPL_AUX_TILE_TIMER_SET_3" }, - { 5892, "MPL_AUX_TILE_TIMER" }, - { 5893, "AUX_TILE_TIMER_CONTROL" }, - { 6144, "MPL_IDN_TIMER_SET_0" }, - { 6145, "MPL_IDN_TIMER_SET_1" }, - { 6146, "MPL_IDN_TIMER_SET_2" }, - { 6147, "MPL_IDN_TIMER_SET_3" }, - { 6148, "MPL_IDN_TIMER" }, - { 6149, "IDN_DEADLOCK_COUNT" }, - { 6150, "IDN_DEADLOCK_TIMEOUT" }, - { 6400, "MPL_UDN_TIMER_SET_0" }, - { 6401, "MPL_UDN_TIMER_SET_1" }, - { 6402, "MPL_UDN_TIMER_SET_2" }, - { 6403, "MPL_UDN_TIMER_SET_3" }, - { 6404, "MPL_UDN_TIMER" }, - { 6405, "UDN_DEADLOCK_COUNT" }, - { 6406, "UDN_DEADLOCK_TIMEOUT" }, - { 6656, "MPL_IDN_AVAIL_SET_0" }, - { 6657, "MPL_IDN_AVAIL_SET_1" }, - { 6658, "MPL_IDN_AVAIL_SET_2" }, - { 6659, "MPL_IDN_AVAIL_SET_3" }, - { 6660, "MPL_IDN_AVAIL" }, - { 6661, "IDN_AVAIL_EN" }, - { 6912, "MPL_UDN_AVAIL_SET_0" }, - { 6913, "MPL_UDN_AVAIL_SET_1" }, - { 6914, "MPL_UDN_AVAIL_SET_2" }, - { 6915, "MPL_UDN_AVAIL_SET_3" }, - { 6916, "MPL_UDN_AVAIL" }, - { 6917, "UDN_AVAIL_EN" }, - { 7168, "MPL_IPI_3_SET_0" }, - { 7169, "MPL_IPI_3_SET_1" }, - { 7170, "MPL_IPI_3_SET_2" }, - { 7171, "MPL_IPI_3_SET_3" }, - { 7172, "MPL_IPI_3" }, - { 7173, "IPI_EVENT_3" }, - { 7174, "IPI_EVENT_RESET_3" }, - { 7175, "IPI_EVENT_SET_3" }, - { 7176, "IPI_MASK_3" }, - { 7177, "IPI_MASK_RESET_3" }, - { 7178, "IPI_MASK_SET_3" }, - { 7424, "MPL_IPI_2_SET_0" }, - { 7425, "MPL_IPI_2_SET_1" }, - { 7426, "MPL_IPI_2_SET_2" }, - { 7427, "MPL_IPI_2_SET_3" }, - { 7428, "MPL_IPI_2" }, - { 7429, "IPI_EVENT_2" }, - { 7430, "IPI_EVENT_RESET_2" }, - { 7431, "IPI_EVENT_SET_2" }, - { 7432, "IPI_MASK_2" }, - { 7433, "IPI_MASK_RESET_2" }, - { 7434, "IPI_MASK_SET_2" }, - { 7680, "MPL_IPI_1_SET_0" }, - { 7681, "MPL_IPI_1_SET_1" }, - { 7682, "MPL_IPI_1_SET_2" }, - { 7683, "MPL_IPI_1_SET_3" }, - { 7684, "MPL_IPI_1" }, - { 7685, "IPI_EVENT_1" }, - { 7686, "IPI_EVENT_RESET_1" }, - { 7687, "IPI_EVENT_SET_1" }, - { 7688, "IPI_MASK_1" }, - { 7689, "IPI_MASK_RESET_1" }, - { 7690, "IPI_MASK_SET_1" }, - { 7936, "MPL_IPI_0_SET_0" }, - { 7937, "MPL_IPI_0_SET_1" }, - { 7938, "MPL_IPI_0_SET_2" }, - { 7939, "MPL_IPI_0_SET_3" }, - { 7940, "MPL_IPI_0" }, - { 7941, "IPI_EVENT_0" }, - { 7942, "IPI_EVENT_RESET_0" }, - { 7943, "IPI_EVENT_SET_0" }, - { 7944, "IPI_MASK_0" }, - { 7945, "IPI_MASK_RESET_0" }, - { 7946, "IPI_MASK_SET_0" }, - { 8192, "MPL_PERF_COUNT_SET_0" }, - { 8193, "MPL_PERF_COUNT_SET_1" }, - { 8194, "MPL_PERF_COUNT_SET_2" }, - { 8195, "MPL_PERF_COUNT_SET_3" }, - { 8196, "MPL_PERF_COUNT" }, - { 8197, "PERF_COUNT_0" }, - { 8198, "PERF_COUNT_1" }, - { 8199, "PERF_COUNT_CTL" }, - { 8200, "PERF_COUNT_DN_CTL" }, - { 8201, "PERF_COUNT_STS" }, - { 8202, "WATCH_MASK" }, - { 8203, "WATCH_VAL" }, - { 8448, "MPL_AUX_PERF_COUNT_SET_0" }, - { 8449, "MPL_AUX_PERF_COUNT_SET_1" }, - { 8450, "MPL_AUX_PERF_COUNT_SET_2" }, - { 8451, "MPL_AUX_PERF_COUNT_SET_3" }, - { 8452, "MPL_AUX_PERF_COUNT" }, - { 8453, "AUX_PERF_COUNT_0" }, - { 8454, "AUX_PERF_COUNT_1" }, - { 8455, "AUX_PERF_COUNT_CTL" }, - { 8456, "AUX_PERF_COUNT_STS" }, - { 8704, "MPL_INTCTRL_3_SET_0" }, - { 8705, "MPL_INTCTRL_3_SET_1" }, - { 8706, "MPL_INTCTRL_3_SET_2" }, - { 8707, "MPL_INTCTRL_3_SET_3" }, - { 8708, "MPL_INTCTRL_3" }, - { 8709, "INTCTRL_3_STATUS" }, - { 8710, "INTERRUPT_MASK_3" }, - { 8711, "INTERRUPT_MASK_RESET_3" }, - { 8712, "INTERRUPT_MASK_SET_3" }, - { 8713, "INTERRUPT_VECTOR_BASE_3" }, - { 8714, "SINGLE_STEP_EN_0_3" }, - { 8715, "SINGLE_STEP_EN_1_3" }, - { 8716, "SINGLE_STEP_EN_2_3" }, - { 8717, "SINGLE_STEP_EN_3_3" }, - { 8832, "EX_CONTEXT_3_0" }, - { 8833, "EX_CONTEXT_3_1" }, - { 8834, "SYSTEM_SAVE_3_0" }, - { 8835, "SYSTEM_SAVE_3_1" }, - { 8836, "SYSTEM_SAVE_3_2" }, - { 8837, "SYSTEM_SAVE_3_3" }, - { 8960, "MPL_INTCTRL_2_SET_0" }, - { 8961, "MPL_INTCTRL_2_SET_1" }, - { 8962, "MPL_INTCTRL_2_SET_2" }, - { 8963, "MPL_INTCTRL_2_SET_3" }, - { 8964, "MPL_INTCTRL_2" }, - { 8965, "INTCTRL_2_STATUS" }, - { 8966, "INTERRUPT_MASK_2" }, - { 8967, "INTERRUPT_MASK_RESET_2" }, - { 8968, "INTERRUPT_MASK_SET_2" }, - { 8969, "INTERRUPT_VECTOR_BASE_2" }, - { 8970, "SINGLE_STEP_EN_0_2" }, - { 8971, "SINGLE_STEP_EN_1_2" }, - { 8972, "SINGLE_STEP_EN_2_2" }, - { 8973, "SINGLE_STEP_EN_3_2" }, - { 9088, "EX_CONTEXT_2_0" }, - { 9089, "EX_CONTEXT_2_1" }, - { 9090, "SYSTEM_SAVE_2_0" }, - { 9091, "SYSTEM_SAVE_2_1" }, - { 9092, "SYSTEM_SAVE_2_2" }, - { 9093, "SYSTEM_SAVE_2_3" }, - { 9216, "MPL_INTCTRL_1_SET_0" }, - { 9217, "MPL_INTCTRL_1_SET_1" }, - { 9218, "MPL_INTCTRL_1_SET_2" }, - { 9219, "MPL_INTCTRL_1_SET_3" }, - { 9220, "MPL_INTCTRL_1" }, - { 9221, "INTCTRL_1_STATUS" }, - { 9222, "INTERRUPT_MASK_1" }, - { 9223, "INTERRUPT_MASK_RESET_1" }, - { 9224, "INTERRUPT_MASK_SET_1" }, - { 9225, "INTERRUPT_VECTOR_BASE_1" }, - { 9226, "SINGLE_STEP_EN_0_1" }, - { 9227, "SINGLE_STEP_EN_1_1" }, - { 9228, "SINGLE_STEP_EN_2_1" }, - { 9229, "SINGLE_STEP_EN_3_1" }, - { 9344, "EX_CONTEXT_1_0" }, - { 9345, "EX_CONTEXT_1_1" }, - { 9346, "SYSTEM_SAVE_1_0" }, - { 9347, "SYSTEM_SAVE_1_1" }, - { 9348, "SYSTEM_SAVE_1_2" }, - { 9349, "SYSTEM_SAVE_1_3" }, - { 9472, "MPL_INTCTRL_0_SET_0" }, - { 9473, "MPL_INTCTRL_0_SET_1" }, - { 9474, "MPL_INTCTRL_0_SET_2" }, - { 9475, "MPL_INTCTRL_0_SET_3" }, - { 9476, "MPL_INTCTRL_0" }, - { 9477, "INTCTRL_0_STATUS" }, - { 9478, "INTERRUPT_MASK_0" }, - { 9479, "INTERRUPT_MASK_RESET_0" }, - { 9480, "INTERRUPT_MASK_SET_0" }, - { 9481, "INTERRUPT_VECTOR_BASE_0" }, - { 9482, "SINGLE_STEP_EN_0_0" }, - { 9483, "SINGLE_STEP_EN_1_0" }, - { 9484, "SINGLE_STEP_EN_2_0" }, - { 9485, "SINGLE_STEP_EN_3_0" }, - { 9600, "EX_CONTEXT_0_0" }, - { 9601, "EX_CONTEXT_0_1" }, - { 9602, "SYSTEM_SAVE_0_0" }, - { 9603, "SYSTEM_SAVE_0_1" }, - { 9604, "SYSTEM_SAVE_0_2" }, - { 9605, "SYSTEM_SAVE_0_3" }, - { 9728, "MPL_BOOT_ACCESS_SET_0" }, - { 9729, "MPL_BOOT_ACCESS_SET_1" }, - { 9730, "MPL_BOOT_ACCESS_SET_2" }, - { 9731, "MPL_BOOT_ACCESS_SET_3" }, - { 9732, "MPL_BOOT_ACCESS" }, - { 9733, "BIG_ENDIAN_CONFIG" }, - { 9734, "CACHE_INVALIDATION_COMPRESSION_MODE" }, - { 9735, "CACHE_INVALIDATION_MASK_0" }, - { 9736, "CACHE_INVALIDATION_MASK_1" }, - { 9737, "CACHE_INVALIDATION_MASK_2" }, - { 9738, "CBOX_CACHEASRAM_CONFIG" }, - { 9739, "CBOX_CACHE_CONFIG" }, - { 9740, "CBOX_HOME_MAP_ADDR" }, - { 9741, "CBOX_HOME_MAP_DATA" }, - { 9742, "CBOX_MMAP_0" }, - { 9743, "CBOX_MMAP_1" }, - { 9744, "CBOX_MMAP_2" }, - { 9745, "CBOX_MMAP_3" }, - { 9746, "CBOX_MSR" }, - { 9747, "DIAG_BCST_CTL" }, - { 9748, "DIAG_BCST_MASK" }, - { 9749, "DIAG_BCST_TRIGGER" }, - { 9750, "DIAG_MUX_CTL" }, - { 9751, "DIAG_TRACE_CTL" }, - { 9752, "DIAG_TRACE_DATA" }, - { 9753, "DIAG_TRACE_STS" }, - { 9754, "IDN_DEMUX_BUF_THRESH" }, - { 9755, "L1_I_PIN_WAY_0" }, - { 9756, "MEM_ROUTE_ORDER" }, - { 9757, "MEM_STRIPE_CONFIG" }, - { 9758, "PERF_COUNT_PLS" }, - { 9759, "PSEUDO_RANDOM_NUMBER_MODIFY" }, - { 9760, "QUIESCE_CTL" }, - { 9761, "RSHIM_COORD" }, - { 9762, "SBOX_CONFIG" }, - { 9763, "UDN_DEMUX_BUF_THRESH" }, - { 9764, "XDN_CORE_STARVATION_COUNT" }, - { 9765, "XDN_ROUND_ROBIN_ARB_CTL" }, - { 9856, "CYCLE_MODIFY" }, - { 9857, "I_AAR" }, - { 9984, "MPL_WORLD_ACCESS_SET_0" }, - { 9985, "MPL_WORLD_ACCESS_SET_1" }, - { 9986, "MPL_WORLD_ACCESS_SET_2" }, - { 9987, "MPL_WORLD_ACCESS_SET_3" }, - { 9988, "MPL_WORLD_ACCESS" }, - { 9989, "DONE" }, - { 9990, "DSTREAM_PF" }, - { 9991, "FAIL" }, - { 9992, "INTERRUPT_CRITICAL_SECTION" }, - { 9993, "PASS" }, - { 9994, "PSEUDO_RANDOM_NUMBER" }, - { 9995, "TILE_COORD" }, - { 9996, "TILE_RTF_HWM" }, - { 10112, "CMPEXCH_VALUE" }, - { 10113, "CYCLE" }, - { 10114, "EVENT_BEGIN" }, - { 10115, "EVENT_END" }, - { 10116, "PROC_STATUS" }, - { 10117, "SIM_CONTROL" }, - { 10118, "SIM_SOCKET" }, - { 10119, "STATUS_SATURATE" }, - { 10240, "MPL_I_ASID_SET_0" }, - { 10241, "MPL_I_ASID_SET_1" }, - { 10242, "MPL_I_ASID_SET_2" }, - { 10243, "MPL_I_ASID_SET_3" }, - { 10244, "MPL_I_ASID" }, - { 10245, "I_ASID" }, - { 10496, "MPL_D_ASID_SET_0" }, - { 10497, "MPL_D_ASID_SET_1" }, - { 10498, "MPL_D_ASID_SET_2" }, - { 10499, "MPL_D_ASID_SET_3" }, - { 10500, "MPL_D_ASID" }, - { 10501, "D_ASID" }, - { 10752, "MPL_DOUBLE_FAULT_SET_0" }, - { 10753, "MPL_DOUBLE_FAULT_SET_1" }, - { 10754, "MPL_DOUBLE_FAULT_SET_2" }, - { 10755, "MPL_DOUBLE_FAULT_SET_3" }, - { 10756, "MPL_DOUBLE_FAULT" }, - { 10757, "LAST_INTERRUPT_REASON" }, -}; - -const int tilegx_num_sprs = 441; - -const char * -get_tilegx_spr_name (int num) -{ - void *result; - struct tilegx_spr key; - - key.number = num; - result = bsearch((const void *) &key, (const void *) tilegx_sprs, - tilegx_num_sprs, sizeof (struct tilegx_spr), - tilegx_spr_compare); - - if (result == NULL) - { - return (NULL); - } - else - { - struct tilegx_spr *result_ptr = (struct tilegx_spr *) result; - return (result_ptr->name); - } -} - -int -print_insn_tilegx (unsigned char * memaddr) -{ - struct tilegx_decoded_instruction - decoded[TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE]; - unsigned char opbuf[TILEGX_BUNDLE_SIZE_IN_BYTES]; - int i, num_instructions, num_printed; - tilegx_mnemonic padding_mnemonic; - - memcpy((void *)opbuf, (void *)memaddr, TILEGX_BUNDLE_SIZE_IN_BYTES); - - /* Parse the instructions in the bundle. */ - num_instructions = - parse_insn_tilegx (*(unsigned long long *)opbuf, (unsigned long long)memaddr, decoded); - - /* Print the instructions in the bundle. */ - printf("{ "); - num_printed = 0; - - /* Determine which nop opcode is used for padding and should be skipped. */ - padding_mnemonic = TILEGX_OPC_FNOP; - for (i = 0; i < num_instructions; i++) - { - if (!decoded[i].opcode->can_bundle) - { - /* Instructions that cannot be bundled are padded out with nops, - rather than fnops. Displaying them is always clutter. */ - padding_mnemonic = TILEGX_OPC_NOP; - break; - } - } - - for (i = 0; i < num_instructions; i++) - { - const struct tilegx_opcode *opcode = decoded[i].opcode; - const char *name; - int j; - - /* Do not print out fnops, unless everything is an fnop, in - which case we will print out just the last one. */ - if (opcode->mnemonic == padding_mnemonic - && (num_printed > 0 || i + 1 < num_instructions)) - continue; - - if (num_printed > 0) - printf(" ; "); - ++num_printed; - - name = opcode->name; - if (name == NULL) - name = "<invalid>"; - printf("%s", name); - - for (j = 0; j < opcode->num_operands; j++) - { - unsigned long long num; - const struct tilegx_operand *op; - const char *spr_name; - - if (j > 0) - printf (","); - printf (" "); - - num = decoded[i].operand_values[j]; - - op = decoded[i].operands[j]; - switch (op->type) - { - case TILEGX_OP_TYPE_REGISTER: - printf ("%s", tilegx_register_names[(int)num]); - break; - case TILEGX_OP_TYPE_SPR: - spr_name = get_tilegx_spr_name(num); - if (spr_name != NULL) - printf ("%s", spr_name); - else - printf ("%d", (int)num); - break; - case TILEGX_OP_TYPE_IMMEDIATE: - printf ("%d", (int)num); - break; - case TILEGX_OP_TYPE_ADDRESS: - printf ("0x%016llx", num); - break; - default: - abort (); - } - } - } - printf (" }\n"); - - return TILEGX_BUNDLE_SIZE_IN_BYTES; -} diff --git a/dist2/src/sljit/sljitNativeTILEGX_64.c b/dist2/src/sljit/sljitNativeTILEGX_64.c deleted file mode 100644 index 003f43a7..00000000 --- a/dist2/src/sljit/sljitNativeTILEGX_64.c +++ /dev/null @@ -1,2555 +0,0 @@ -/* - * Stack-less Just-In-Time compiler - * - * Copyright 2013-2013 Tilera Corporation(jiwang@tilera.com). All rights reserved. - * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are - * permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, this list - * of conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* TileGX architecture. */ -/* Contributed by Tilera Corporation. */ -#include "sljitNativeTILEGX-encoder.c" - -#define SIMM_8BIT_MAX (0x7f) -#define SIMM_8BIT_MIN (-0x80) -#define SIMM_16BIT_MAX (0x7fff) -#define SIMM_16BIT_MIN (-0x8000) -#define SIMM_17BIT_MAX (0xffff) -#define SIMM_17BIT_MIN (-0x10000) -#define SIMM_32BIT_MAX (0x7fffffff) -#define SIMM_32BIT_MIN (-0x7fffffff - 1) -#define SIMM_48BIT_MAX (0x7fffffff0000L) -#define SIMM_48BIT_MIN (-0x800000000000L) -#define IMM16(imm) ((imm) & 0xffff) - -#define UIMM_16BIT_MAX (0xffff) - -#define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2) -#define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3) -#define TMP_REG3 (SLJIT_NUMBER_OF_REGISTERS + 4) -#define ADDR_TMP (SLJIT_NUMBER_OF_REGISTERS + 5) -#define PIC_ADDR_REG TMP_REG2 - -static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 6] = { - 63, 0, 1, 2, 3, 4, 30, 31, 32, 33, 34, 54, 5, 16, 6, 7 -}; - -#define SLJIT_LOCALS_REG_mapped 54 -#define TMP_REG1_mapped 5 -#define TMP_REG2_mapped 16 -#define TMP_REG3_mapped 6 -#define ADDR_TMP_mapped 7 - -/* Flags are keept in volatile registers. */ -#define EQUAL_FLAG 8 -/* And carry flag as well. */ -#define ULESS_FLAG 9 -#define UGREATER_FLAG 10 -#define LESS_FLAG 11 -#define GREATER_FLAG 12 -#define OVERFLOW_FLAG 13 - -#define ZERO 63 -#define RA 55 -#define TMP_EREG1 14 -#define TMP_EREG2 15 - -#define LOAD_DATA 0x01 -#define WORD_DATA 0x00 -#define BYTE_DATA 0x02 -#define HALF_DATA 0x04 -#define INT_DATA 0x06 -#define SIGNED_DATA 0x08 -#define DOUBLE_DATA 0x10 - -/* Separates integer and floating point registers */ -#define GPR_REG 0xf - -#define MEM_MASK 0x1f - -#define WRITE_BACK 0x00020 -#define ARG_TEST 0x00040 -#define ALT_KEEP_CACHE 0x00080 -#define CUMULATIVE_OP 0x00100 -#define LOGICAL_OP 0x00200 -#define IMM_OP 0x00400 -#define SRC2_IMM 0x00800 - -#define UNUSED_DEST 0x01000 -#define REG_DEST 0x02000 -#define REG1_SOURCE 0x04000 -#define REG2_SOURCE 0x08000 -#define SLOW_SRC1 0x10000 -#define SLOW_SRC2 0x20000 -#define SLOW_DEST 0x40000 - -/* Only these flags are set. UNUSED_DEST is not set when no flags should be set. - */ -#define CHECK_FLAGS(list) (!(flags & UNUSED_DEST) || (op & GET_FLAGS(~(list)))) - -SLJIT_API_FUNC_ATTRIBUTE const char *sljit_get_platform_name(void) -{ - return "TileGX" SLJIT_CPUINFO; -} - -/* Length of an instruction word */ -typedef sljit_uw sljit_ins; - -struct jit_instr { - const struct tilegx_opcode* opcode; - tilegx_pipeline pipe; - unsigned long input_registers; - unsigned long output_registers; - int operand_value[4]; - int line; -}; - -/* Opcode Helper Macros */ -#define TILEGX_X_MODE 0 - -#define X_MODE create_Mode(TILEGX_X_MODE) - -#define FNOP_X0 \ - create_Opcode_X0(RRR_0_OPCODE_X0) | \ - create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | \ - create_UnaryOpcodeExtension_X0(FNOP_UNARY_OPCODE_X0) - -#define FNOP_X1 \ - create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | \ - create_UnaryOpcodeExtension_X1(FNOP_UNARY_OPCODE_X1) - -#define NOP \ - create_Mode(TILEGX_X_MODE) | FNOP_X0 | FNOP_X1 - -#define ANOP_X0 \ - create_Opcode_X0(RRR_0_OPCODE_X0) | \ - create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | \ - create_UnaryOpcodeExtension_X0(NOP_UNARY_OPCODE_X0) - -#define BPT create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | \ - create_UnaryOpcodeExtension_X1(ILL_UNARY_OPCODE_X1) | \ - create_Dest_X1(0x1C) | create_SrcA_X1(0x25) | ANOP_X0 - -#define ADD_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(ADD_RRR_0_OPCODE_X1) | FNOP_X0 - -#define ADDI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(IMM8_OPCODE_X1) | \ - create_Imm8OpcodeExtension_X1(ADDI_IMM8_OPCODE_X1) | FNOP_X0 - -#define SUB_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(SUB_RRR_0_OPCODE_X1) | FNOP_X0 - -#define NOR_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(NOR_RRR_0_OPCODE_X1) | FNOP_X0 - -#define OR_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(OR_RRR_0_OPCODE_X1) | FNOP_X0 - -#define AND_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(AND_RRR_0_OPCODE_X1) | FNOP_X0 - -#define XOR_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(XOR_RRR_0_OPCODE_X1) | FNOP_X0 - -#define CMOVNEZ_X0 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X0(RRR_0_OPCODE_X0) | \ - create_RRROpcodeExtension_X0(CMOVNEZ_RRR_0_OPCODE_X0) | FNOP_X1 - -#define CMOVEQZ_X0 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X0(RRR_0_OPCODE_X0) | \ - create_RRROpcodeExtension_X0(CMOVEQZ_RRR_0_OPCODE_X0) | FNOP_X1 - -#define ADDLI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(ADDLI_OPCODE_X1) | FNOP_X0 - -#define V4INT_L_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(V4INT_L_RRR_0_OPCODE_X1) | FNOP_X0 - -#define BFEXTU_X0 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X0(BF_OPCODE_X0) | \ - create_BFOpcodeExtension_X0(BFEXTU_BF_OPCODE_X0) | FNOP_X1 - -#define BFEXTS_X0 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X0(BF_OPCODE_X0) | \ - create_BFOpcodeExtension_X0(BFEXTS_BF_OPCODE_X0) | FNOP_X1 - -#define SHL16INSLI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(SHL16INSLI_OPCODE_X1) | FNOP_X0 - -#define ST_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(ST_RRR_0_OPCODE_X1) | create_Dest_X1(0x0) | FNOP_X0 - -#define LD_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | \ - create_UnaryOpcodeExtension_X1(LD_UNARY_OPCODE_X1) | FNOP_X0 - -#define JR_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | \ - create_UnaryOpcodeExtension_X1(JR_UNARY_OPCODE_X1) | FNOP_X0 - -#define JALR_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | \ - create_UnaryOpcodeExtension_X1(JALR_UNARY_OPCODE_X1) | FNOP_X0 - -#define CLZ_X0 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X0(RRR_0_OPCODE_X0) | \ - create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | \ - create_UnaryOpcodeExtension_X0(CNTLZ_UNARY_OPCODE_X0) | FNOP_X1 - -#define CMPLTUI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(IMM8_OPCODE_X1) | \ - create_Imm8OpcodeExtension_X1(CMPLTUI_IMM8_OPCODE_X1) | FNOP_X0 - -#define CMPLTU_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(CMPLTU_RRR_0_OPCODE_X1) | FNOP_X0 - -#define CMPLTS_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(CMPLTS_RRR_0_OPCODE_X1) | FNOP_X0 - -#define XORI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(IMM8_OPCODE_X1) | \ - create_Imm8OpcodeExtension_X1(XORI_IMM8_OPCODE_X1) | FNOP_X0 - -#define ORI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(IMM8_OPCODE_X1) | \ - create_Imm8OpcodeExtension_X1(ORI_IMM8_OPCODE_X1) | FNOP_X0 - -#define ANDI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(IMM8_OPCODE_X1) | \ - create_Imm8OpcodeExtension_X1(ANDI_IMM8_OPCODE_X1) | FNOP_X0 - -#define SHLI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(SHIFT_OPCODE_X1) | \ - create_ShiftOpcodeExtension_X1(SHLI_SHIFT_OPCODE_X1) | FNOP_X0 - -#define SHL_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(SHL_RRR_0_OPCODE_X1) | FNOP_X0 - -#define SHRSI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(SHIFT_OPCODE_X1) | \ - create_ShiftOpcodeExtension_X1(SHRSI_SHIFT_OPCODE_X1) | FNOP_X0 - -#define SHRS_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(SHRS_RRR_0_OPCODE_X1) | FNOP_X0 - -#define SHRUI_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(SHIFT_OPCODE_X1) | \ - create_ShiftOpcodeExtension_X1(SHRUI_SHIFT_OPCODE_X1) | FNOP_X0 - -#define SHRU_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(RRR_0_OPCODE_X1) | \ - create_RRROpcodeExtension_X1(SHRU_RRR_0_OPCODE_X1) | FNOP_X0 - -#define BEQZ_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(BRANCH_OPCODE_X1) | \ - create_BrType_X1(BEQZ_BRANCH_OPCODE_X1) | FNOP_X0 - -#define BNEZ_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(BRANCH_OPCODE_X1) | \ - create_BrType_X1(BNEZ_BRANCH_OPCODE_X1) | FNOP_X0 - -#define J_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(JUMP_OPCODE_X1) | \ - create_JumpOpcodeExtension_X1(J_JUMP_OPCODE_X1) | FNOP_X0 - -#define JAL_X1 \ - create_Mode(TILEGX_X_MODE) | create_Opcode_X1(JUMP_OPCODE_X1) | \ - create_JumpOpcodeExtension_X1(JAL_JUMP_OPCODE_X1) | FNOP_X0 - -#define DEST_X0(x) create_Dest_X0(x) -#define SRCA_X0(x) create_SrcA_X0(x) -#define SRCB_X0(x) create_SrcB_X0(x) -#define DEST_X1(x) create_Dest_X1(x) -#define SRCA_X1(x) create_SrcA_X1(x) -#define SRCB_X1(x) create_SrcB_X1(x) -#define IMM16_X1(x) create_Imm16_X1(x) -#define IMM8_X1(x) create_Imm8_X1(x) -#define BFSTART_X0(x) create_BFStart_X0(x) -#define BFEND_X0(x) create_BFEnd_X0(x) -#define SHIFTIMM_X1(x) create_ShAmt_X1(x) -#define JOFF_X1(x) create_JumpOff_X1(x) -#define BOFF_X1(x) create_BrOff_X1(x) - -static const tilegx_mnemonic data_transfer_insts[16] = { - /* u w s */ TILEGX_OPC_ST /* st */, - /* u w l */ TILEGX_OPC_LD /* ld */, - /* u b s */ TILEGX_OPC_ST1 /* st1 */, - /* u b l */ TILEGX_OPC_LD1U /* ld1u */, - /* u h s */ TILEGX_OPC_ST2 /* st2 */, - /* u h l */ TILEGX_OPC_LD2U /* ld2u */, - /* u i s */ TILEGX_OPC_ST4 /* st4 */, - /* u i l */ TILEGX_OPC_LD4U /* ld4u */, - /* s w s */ TILEGX_OPC_ST /* st */, - /* s w l */ TILEGX_OPC_LD /* ld */, - /* s b s */ TILEGX_OPC_ST1 /* st1 */, - /* s b l */ TILEGX_OPC_LD1S /* ld1s */, - /* s h s */ TILEGX_OPC_ST2 /* st2 */, - /* s h l */ TILEGX_OPC_LD2S /* ld2s */, - /* s i s */ TILEGX_OPC_ST4 /* st4 */, - /* s i l */ TILEGX_OPC_LD4S /* ld4s */, -}; - -#ifdef TILEGX_JIT_DEBUG -static sljit_s32 push_inst_debug(struct sljit_compiler *compiler, sljit_ins ins, int line) -{ - sljit_ins *ptr = (sljit_ins *)ensure_buf(compiler, sizeof(sljit_ins)); - FAIL_IF(!ptr); - *ptr = ins; - compiler->size++; - printf("|%04d|S0|:\t\t", line); - print_insn_tilegx(ptr); - return SLJIT_SUCCESS; -} - -static sljit_s32 push_inst_nodebug(struct sljit_compiler *compiler, sljit_ins ins) -{ - sljit_ins *ptr = (sljit_ins *)ensure_buf(compiler, sizeof(sljit_ins)); - FAIL_IF(!ptr); - *ptr = ins; - compiler->size++; - return SLJIT_SUCCESS; -} - -#define push_inst(a, b) push_inst_debug(a, b, __LINE__) -#else -static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins) -{ - sljit_ins *ptr = (sljit_ins *)ensure_buf(compiler, sizeof(sljit_ins)); - FAIL_IF(!ptr); - *ptr = ins; - compiler->size++; - return SLJIT_SUCCESS; -} -#endif - -#define BUNDLE_FORMAT_MASK(p0, p1, p2) \ - ((p0) | ((p1) << 8) | ((p2) << 16)) - -#define BUNDLE_FORMAT(p0, p1, p2) \ - { \ - { \ - (tilegx_pipeline)(p0), \ - (tilegx_pipeline)(p1), \ - (tilegx_pipeline)(p2) \ - }, \ - BUNDLE_FORMAT_MASK(1 << (p0), 1 << (p1), (1 << (p2))) \ - } - -#define NO_PIPELINE TILEGX_NUM_PIPELINE_ENCODINGS - -#define tilegx_is_x_pipeline(p) ((int)(p) <= (int)TILEGX_PIPELINE_X1) - -#define PI(encoding) \ - push_inst(compiler, encoding) - -#define PB3(opcode, dst, srca, srcb) \ - push_3_buffer(compiler, opcode, dst, srca, srcb, __LINE__) - -#define PB2(opcode, dst, src) \ - push_2_buffer(compiler, opcode, dst, src, __LINE__) - -#define JR(reg) \ - push_jr_buffer(compiler, TILEGX_OPC_JR, reg, __LINE__) - -#define ADD(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_ADD, dst, srca, srcb, __LINE__) - -#define SUB(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_SUB, dst, srca, srcb, __LINE__) - -#define MUL(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_MULX, dst, srca, srcb, __LINE__) - -#define NOR(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_NOR, dst, srca, srcb, __LINE__) - -#define OR(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_OR, dst, srca, srcb, __LINE__) - -#define XOR(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_XOR, dst, srca, srcb, __LINE__) - -#define AND(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_AND, dst, srca, srcb, __LINE__) - -#define CLZ(dst, src) \ - push_2_buffer(compiler, TILEGX_OPC_CLZ, dst, src, __LINE__) - -#define SHLI(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_SHLI, dst, srca, srcb, __LINE__) - -#define SHRUI(dst, srca, imm) \ - push_3_buffer(compiler, TILEGX_OPC_SHRUI, dst, srca, imm, __LINE__) - -#define XORI(dst, srca, imm) \ - push_3_buffer(compiler, TILEGX_OPC_XORI, dst, srca, imm, __LINE__) - -#define ORI(dst, srca, imm) \ - push_3_buffer(compiler, TILEGX_OPC_ORI, dst, srca, imm, __LINE__) - -#define CMPLTU(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_CMPLTU, dst, srca, srcb, __LINE__) - -#define CMPLTS(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_CMPLTS, dst, srca, srcb, __LINE__) - -#define CMPLTUI(dst, srca, imm) \ - push_3_buffer(compiler, TILEGX_OPC_CMPLTUI, dst, srca, imm, __LINE__) - -#define CMOVNEZ(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_CMOVNEZ, dst, srca, srcb, __LINE__) - -#define CMOVEQZ(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_CMOVEQZ, dst, srca, srcb, __LINE__) - -#define ADDLI(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_ADDLI, dst, srca, srcb, __LINE__) - -#define SHL16INSLI(dst, srca, srcb) \ - push_3_buffer(compiler, TILEGX_OPC_SHL16INSLI, dst, srca, srcb, __LINE__) - -#define LD_ADD(dst, addr, adjust) \ - push_3_buffer(compiler, TILEGX_OPC_LD_ADD, dst, addr, adjust, __LINE__) - -#define ST_ADD(src, addr, adjust) \ - push_3_buffer(compiler, TILEGX_OPC_ST_ADD, src, addr, adjust, __LINE__) - -#define LD(dst, addr) \ - push_2_buffer(compiler, TILEGX_OPC_LD, dst, addr, __LINE__) - -#define BFEXTU(dst, src, start, end) \ - push_4_buffer(compiler, TILEGX_OPC_BFEXTU, dst, src, start, end, __LINE__) - -#define BFEXTS(dst, src, start, end) \ - push_4_buffer(compiler, TILEGX_OPC_BFEXTS, dst, src, start, end, __LINE__) - -#define ADD_SOLO(dest, srca, srcb) \ - push_inst(compiler, ADD_X1 | DEST_X1(dest) | SRCA_X1(srca) | SRCB_X1(srcb)) - -#define ADDI_SOLO(dest, srca, imm) \ - push_inst(compiler, ADDI_X1 | DEST_X1(dest) | SRCA_X1(srca) | IMM8_X1(imm)) - -#define ADDLI_SOLO(dest, srca, imm) \ - push_inst(compiler, ADDLI_X1 | DEST_X1(dest) | SRCA_X1(srca) | IMM16_X1(imm)) - -#define SHL16INSLI_SOLO(dest, srca, imm) \ - push_inst(compiler, SHL16INSLI_X1 | DEST_X1(dest) | SRCA_X1(srca) | IMM16_X1(imm)) - -#define JALR_SOLO(reg) \ - push_inst(compiler, JALR_X1 | SRCA_X1(reg)) - -#define JR_SOLO(reg) \ - push_inst(compiler, JR_X1 | SRCA_X1(reg)) - -struct Format { - /* Mapping of bundle issue slot to assigned pipe. */ - tilegx_pipeline pipe[TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE]; - - /* Mask of pipes used by this bundle. */ - unsigned int pipe_mask; -}; - -const struct Format formats[] = -{ - /* In Y format we must always have something in Y2, since it has - * no fnop, so this conveys that Y2 must always be used. */ - BUNDLE_FORMAT(TILEGX_PIPELINE_Y0, TILEGX_PIPELINE_Y2, NO_PIPELINE), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y1, TILEGX_PIPELINE_Y2, NO_PIPELINE), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y0, NO_PIPELINE), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y1, NO_PIPELINE), - - /* Y format has three instructions. */ - BUNDLE_FORMAT(TILEGX_PIPELINE_Y0, TILEGX_PIPELINE_Y1, TILEGX_PIPELINE_Y2), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y0, TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y1), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y1, TILEGX_PIPELINE_Y0, TILEGX_PIPELINE_Y2), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y1, TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y0), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y0, TILEGX_PIPELINE_Y1), - BUNDLE_FORMAT(TILEGX_PIPELINE_Y2, TILEGX_PIPELINE_Y1, TILEGX_PIPELINE_Y0), - - /* X format has only two instructions. */ - BUNDLE_FORMAT(TILEGX_PIPELINE_X0, TILEGX_PIPELINE_X1, NO_PIPELINE), - BUNDLE_FORMAT(TILEGX_PIPELINE_X1, TILEGX_PIPELINE_X0, NO_PIPELINE) -}; - - -struct jit_instr inst_buf[TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE]; -unsigned long inst_buf_index; - -tilegx_pipeline get_any_valid_pipe(const struct tilegx_opcode* opcode) -{ - /* FIXME: tile: we could pregenerate this. */ - int pipe; - for (pipe = 0; ((opcode->pipes & (1 << pipe)) == 0 && pipe < TILEGX_NUM_PIPELINE_ENCODINGS); pipe++) - ; - return (tilegx_pipeline)(pipe); -} - -void insert_nop(tilegx_mnemonic opc, int line) -{ - const struct tilegx_opcode* opcode = NULL; - - memmove(&inst_buf[1], &inst_buf[0], inst_buf_index * sizeof inst_buf[0]); - - opcode = &tilegx_opcodes[opc]; - inst_buf[0].opcode = opcode; - inst_buf[0].pipe = get_any_valid_pipe(opcode); - inst_buf[0].input_registers = 0; - inst_buf[0].output_registers = 0; - inst_buf[0].line = line; - ++inst_buf_index; -} - -const struct Format* compute_format() -{ - unsigned int compatible_pipes = BUNDLE_FORMAT_MASK( - inst_buf[0].opcode->pipes, - inst_buf[1].opcode->pipes, - (inst_buf_index == 3 ? inst_buf[2].opcode->pipes : (1 << NO_PIPELINE))); - - const struct Format* match = NULL; - const struct Format *b = NULL; - unsigned int i; - for (i = 0; i < sizeof formats / sizeof formats[0]; i++) { - b = &formats[i]; - if ((b->pipe_mask & compatible_pipes) == b->pipe_mask) { - match = b; - break; - } - } - - return match; -} - -sljit_s32 assign_pipes() -{ - unsigned long output_registers = 0; - unsigned int i = 0; - - if (inst_buf_index == 1) { - tilegx_mnemonic opc = inst_buf[0].opcode->can_bundle - ? TILEGX_OPC_FNOP : TILEGX_OPC_NOP; - insert_nop(opc, __LINE__); - } - - const struct Format* match = compute_format(); - - if (match == NULL) - return -1; - - for (i = 0; i < inst_buf_index; i++) { - - if ((i > 0) && ((inst_buf[i].input_registers & output_registers) != 0)) - return -1; - - if ((i > 0) && ((inst_buf[i].output_registers & output_registers) != 0)) - return -1; - - /* Don't include Rzero in the match set, to avoid triggering - needlessly on 'prefetch' instrs. */ - - output_registers |= inst_buf[i].output_registers & 0xFFFFFFFFFFFFFFL; - - inst_buf[i].pipe = match->pipe[i]; - } - - /* If only 2 instrs, and in Y-mode, insert a nop. */ - if (inst_buf_index == 2 && !tilegx_is_x_pipeline(match->pipe[0])) { - insert_nop(TILEGX_OPC_FNOP, __LINE__); - - /* Select the yet unassigned pipe. */ - tilegx_pipeline pipe = (tilegx_pipeline)(((TILEGX_PIPELINE_Y0 - + TILEGX_PIPELINE_Y1 + TILEGX_PIPELINE_Y2) - - (inst_buf[1].pipe + inst_buf[2].pipe))); - - inst_buf[0].pipe = pipe; - } - - return 0; -} - -tilegx_bundle_bits get_bundle_bit(struct jit_instr *inst) -{ - int i, val; - const struct tilegx_opcode* opcode = inst->opcode; - tilegx_bundle_bits bits = opcode->fixed_bit_values[inst->pipe]; - - const struct tilegx_operand* operand = NULL; - for (i = 0; i < opcode->num_operands; i++) { - operand = &tilegx_operands[opcode->operands[inst->pipe][i]]; - val = inst->operand_value[i]; - - bits |= operand->insert(val); - } - - return bits; -} - -static sljit_s32 update_buffer(struct sljit_compiler *compiler) -{ - int i; - int orig_index = inst_buf_index; - struct jit_instr inst0 = inst_buf[0]; - struct jit_instr inst1 = inst_buf[1]; - struct jit_instr inst2 = inst_buf[2]; - tilegx_bundle_bits bits = 0; - - /* If the bundle is valid as is, perform the encoding and return 1. */ - if (assign_pipes() == 0) { - for (i = 0; i < inst_buf_index; i++) { - bits |= get_bundle_bit(inst_buf + i); -#ifdef TILEGX_JIT_DEBUG - printf("|%04d", inst_buf[i].line); -#endif - } -#ifdef TILEGX_JIT_DEBUG - if (inst_buf_index == 3) - printf("|M0|:\t"); - else - printf("|M0|:\t\t"); - print_insn_tilegx(&bits); -#endif - - inst_buf_index = 0; - -#ifdef TILEGX_JIT_DEBUG - return push_inst_nodebug(compiler, bits); -#else - return push_inst(compiler, bits); -#endif - } - - /* If the bundle is invalid, split it in two. First encode the first two - (or possibly 1) instructions, and then the last, separately. Note that - assign_pipes may have re-ordered the instrs (by inserting no-ops in - lower slots) so we need to reset them. */ - - inst_buf_index = orig_index - 1; - inst_buf[0] = inst0; - inst_buf[1] = inst1; - inst_buf[2] = inst2; - if (assign_pipes() == 0) { - for (i = 0; i < inst_buf_index; i++) { - bits |= get_bundle_bit(inst_buf + i); -#ifdef TILEGX_JIT_DEBUG - printf("|%04d", inst_buf[i].line); -#endif - } - -#ifdef TILEGX_JIT_DEBUG - if (inst_buf_index == 3) - printf("|M1|:\t"); - else - printf("|M1|:\t\t"); - print_insn_tilegx(&bits); -#endif - - if ((orig_index - 1) == 2) { - inst_buf[0] = inst2; - inst_buf_index = 1; - } else if ((orig_index - 1) == 1) { - inst_buf[0] = inst1; - inst_buf_index = 1; - } else - SLJIT_UNREACHABLE(); - -#ifdef TILEGX_JIT_DEBUG - return push_inst_nodebug(compiler, bits); -#else - return push_inst(compiler, bits); -#endif - } else { - /* We had 3 instrs of which the first 2 can't live in the same bundle. - Split those two. Note that we don't try to then combine the second - and third instr into a single bundle. First instruction: */ - inst_buf_index = 1; - inst_buf[0] = inst0; - inst_buf[1] = inst1; - inst_buf[2] = inst2; - if (assign_pipes() == 0) { - for (i = 0; i < inst_buf_index; i++) { - bits |= get_bundle_bit(inst_buf + i); -#ifdef TILEGX_JIT_DEBUG - printf("|%04d", inst_buf[i].line); -#endif - } - -#ifdef TILEGX_JIT_DEBUG - if (inst_buf_index == 3) - printf("|M2|:\t"); - else - printf("|M2|:\t\t"); - print_insn_tilegx(&bits); -#endif - - inst_buf[0] = inst1; - inst_buf[1] = inst2; - inst_buf_index = orig_index - 1; -#ifdef TILEGX_JIT_DEBUG - return push_inst_nodebug(compiler, bits); -#else - return push_inst(compiler, bits); -#endif - } else - SLJIT_UNREACHABLE(); - } - - SLJIT_UNREACHABLE(); -} - -static sljit_s32 flush_buffer(struct sljit_compiler *compiler) -{ - while (inst_buf_index != 0) { - FAIL_IF(update_buffer(compiler)); - } - return SLJIT_SUCCESS; -} - -static sljit_s32 push_4_buffer(struct sljit_compiler *compiler, tilegx_mnemonic opc, int op0, int op1, int op2, int op3, int line) -{ - if (inst_buf_index == TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE) - FAIL_IF(update_buffer(compiler)); - - const struct tilegx_opcode* opcode = &tilegx_opcodes[opc]; - inst_buf[inst_buf_index].opcode = opcode; - inst_buf[inst_buf_index].pipe = get_any_valid_pipe(opcode); - inst_buf[inst_buf_index].operand_value[0] = op0; - inst_buf[inst_buf_index].operand_value[1] = op1; - inst_buf[inst_buf_index].operand_value[2] = op2; - inst_buf[inst_buf_index].operand_value[3] = op3; - inst_buf[inst_buf_index].input_registers = 1L << op1; - inst_buf[inst_buf_index].output_registers = 1L << op0; - inst_buf[inst_buf_index].line = line; - inst_buf_index++; - - return SLJIT_SUCCESS; -} - -static sljit_s32 push_3_buffer(struct sljit_compiler *compiler, tilegx_mnemonic opc, int op0, int op1, int op2, int line) -{ - if (inst_buf_index == TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE) - FAIL_IF(update_buffer(compiler)); - - const struct tilegx_opcode* opcode = &tilegx_opcodes[opc]; - inst_buf[inst_buf_index].opcode = opcode; - inst_buf[inst_buf_index].pipe = get_any_valid_pipe(opcode); - inst_buf[inst_buf_index].operand_value[0] = op0; - inst_buf[inst_buf_index].operand_value[1] = op1; - inst_buf[inst_buf_index].operand_value[2] = op2; - inst_buf[inst_buf_index].line = line; - - switch (opc) { - case TILEGX_OPC_ST_ADD: - inst_buf[inst_buf_index].input_registers = (1L << op0) | (1L << op1); - inst_buf[inst_buf_index].output_registers = 1L << op0; - break; - case TILEGX_OPC_LD_ADD: - inst_buf[inst_buf_index].input_registers = 1L << op1; - inst_buf[inst_buf_index].output_registers = (1L << op0) | (1L << op1); - break; - case TILEGX_OPC_ADD: - case TILEGX_OPC_AND: - case TILEGX_OPC_SUB: - case TILEGX_OPC_MULX: - case TILEGX_OPC_OR: - case TILEGX_OPC_XOR: - case TILEGX_OPC_NOR: - case TILEGX_OPC_SHL: - case TILEGX_OPC_SHRU: - case TILEGX_OPC_SHRS: - case TILEGX_OPC_CMPLTU: - case TILEGX_OPC_CMPLTS: - case TILEGX_OPC_CMOVEQZ: - case TILEGX_OPC_CMOVNEZ: - inst_buf[inst_buf_index].input_registers = (1L << op1) | (1L << op2); - inst_buf[inst_buf_index].output_registers = 1L << op0; - break; - case TILEGX_OPC_ADDLI: - case TILEGX_OPC_XORI: - case TILEGX_OPC_ORI: - case TILEGX_OPC_SHLI: - case TILEGX_OPC_SHRUI: - case TILEGX_OPC_SHRSI: - case TILEGX_OPC_SHL16INSLI: - case TILEGX_OPC_CMPLTUI: - case TILEGX_OPC_CMPLTSI: - inst_buf[inst_buf_index].input_registers = 1L << op1; - inst_buf[inst_buf_index].output_registers = 1L << op0; - break; - default: - printf("unrecoginzed opc: %s\n", opcode->name); - SLJIT_UNREACHABLE(); - } - - inst_buf_index++; - - return SLJIT_SUCCESS; -} - -static sljit_s32 push_2_buffer(struct sljit_compiler *compiler, tilegx_mnemonic opc, int op0, int op1, int line) -{ - if (inst_buf_index == TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE) - FAIL_IF(update_buffer(compiler)); - - const struct tilegx_opcode* opcode = &tilegx_opcodes[opc]; - inst_buf[inst_buf_index].opcode = opcode; - inst_buf[inst_buf_index].pipe = get_any_valid_pipe(opcode); - inst_buf[inst_buf_index].operand_value[0] = op0; - inst_buf[inst_buf_index].operand_value[1] = op1; - inst_buf[inst_buf_index].line = line; - - switch (opc) { - case TILEGX_OPC_BEQZ: - case TILEGX_OPC_BNEZ: - inst_buf[inst_buf_index].input_registers = 1L << op0; - break; - case TILEGX_OPC_ST: - case TILEGX_OPC_ST1: - case TILEGX_OPC_ST2: - case TILEGX_OPC_ST4: - inst_buf[inst_buf_index].input_registers = (1L << op0) | (1L << op1); - inst_buf[inst_buf_index].output_registers = 0; - break; - case TILEGX_OPC_CLZ: - case TILEGX_OPC_LD: - case TILEGX_OPC_LD1U: - case TILEGX_OPC_LD1S: - case TILEGX_OPC_LD2U: - case TILEGX_OPC_LD2S: - case TILEGX_OPC_LD4U: - case TILEGX_OPC_LD4S: - inst_buf[inst_buf_index].input_registers = 1L << op1; - inst_buf[inst_buf_index].output_registers = 1L << op0; - break; - default: - printf("unrecoginzed opc: %s\n", opcode->name); - SLJIT_UNREACHABLE(); - } - - inst_buf_index++; - - return SLJIT_SUCCESS; -} - -static sljit_s32 push_0_buffer(struct sljit_compiler *compiler, tilegx_mnemonic opc, int line) -{ - if (inst_buf_index == TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE) - FAIL_IF(update_buffer(compiler)); - - const struct tilegx_opcode* opcode = &tilegx_opcodes[opc]; - inst_buf[inst_buf_index].opcode = opcode; - inst_buf[inst_buf_index].pipe = get_any_valid_pipe(opcode); - inst_buf[inst_buf_index].input_registers = 0; - inst_buf[inst_buf_index].output_registers = 0; - inst_buf[inst_buf_index].line = line; - inst_buf_index++; - - return SLJIT_SUCCESS; -} - -static sljit_s32 push_jr_buffer(struct sljit_compiler *compiler, tilegx_mnemonic opc, int op0, int line) -{ - if (inst_buf_index == TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE) - FAIL_IF(update_buffer(compiler)); - - const struct tilegx_opcode* opcode = &tilegx_opcodes[opc]; - inst_buf[inst_buf_index].opcode = opcode; - inst_buf[inst_buf_index].pipe = get_any_valid_pipe(opcode); - inst_buf[inst_buf_index].operand_value[0] = op0; - inst_buf[inst_buf_index].input_registers = 1L << op0; - inst_buf[inst_buf_index].output_registers = 0; - inst_buf[inst_buf_index].line = line; - inst_buf_index++; - - return flush_buffer(compiler); -} - -static SLJIT_INLINE sljit_ins * detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code) -{ - sljit_sw diff; - sljit_uw target_addr; - sljit_ins *inst; - - if (jump->flags & SLJIT_REWRITABLE_JUMP) - return code_ptr; - - if (jump->flags & JUMP_ADDR) - target_addr = jump->u.target; - else { - SLJIT_ASSERT(jump->flags & JUMP_LABEL); - target_addr = (sljit_uw)(code + jump->u.label->size); - } - - inst = (sljit_ins *)jump->addr; - if (jump->flags & IS_COND) - inst--; - - diff = ((sljit_sw) target_addr - (sljit_sw) inst) >> 3; - if (diff <= SIMM_17BIT_MAX && diff >= SIMM_17BIT_MIN) { - jump->flags |= PATCH_B; - - if (!(jump->flags & IS_COND)) { - if (jump->flags & IS_JAL) { - jump->flags &= ~(PATCH_B); - jump->flags |= PATCH_J; - inst[0] = JAL_X1; - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(inst); -#endif - } else { - inst[0] = BEQZ_X1 | SRCA_X1(ZERO); - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(inst); -#endif - } - - return inst; - } - - inst[0] = inst[0] ^ (0x7L << 55); - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(inst); -#endif - jump->addr -= sizeof(sljit_ins); - return inst; - } - - if (jump->flags & IS_COND) { - if ((target_addr & ~0x3FFFFFFFL) == ((jump->addr + sizeof(sljit_ins)) & ~0x3FFFFFFFL)) { - jump->flags |= PATCH_J; - inst[0] = (inst[0] & ~(BOFF_X1(-1))) | BOFF_X1(2); - inst[1] = J_X1; - return inst + 1; - } - - return code_ptr; - } - - if ((target_addr & ~0x3FFFFFFFL) == ((jump->addr + sizeof(sljit_ins)) & ~0x3FFFFFFFL)) { - jump->flags |= PATCH_J; - - if (jump->flags & IS_JAL) { - inst[0] = JAL_X1; - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(inst); -#endif - - } else { - inst[0] = J_X1; - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(inst); -#endif - } - - return inst; - } - - return code_ptr; -} - -SLJIT_API_FUNC_ATTRIBUTE void * sljit_generate_code(struct sljit_compiler *compiler) -{ - struct sljit_memory_fragment *buf; - sljit_ins *code; - sljit_ins *code_ptr; - sljit_ins *buf_ptr; - sljit_ins *buf_end; - sljit_uw word_count; - sljit_uw addr; - - struct sljit_label *label; - struct sljit_jump *jump; - struct sljit_const *const_; - - CHECK_ERROR_PTR(); - CHECK_PTR(check_sljit_generate_code(compiler)); - reverse_buf(compiler); - - code = (sljit_ins *)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); - PTR_FAIL_WITH_EXEC_IF(code); - buf = compiler->buf; - - code_ptr = code; - word_count = 0; - label = compiler->labels; - jump = compiler->jumps; - const_ = compiler->consts; - do { - buf_ptr = (sljit_ins *)buf->memory; - buf_end = buf_ptr + (buf->used_size >> 3); - do { - *code_ptr = *buf_ptr++; - SLJIT_ASSERT(!label || label->size >= word_count); - SLJIT_ASSERT(!jump || jump->addr >= word_count); - SLJIT_ASSERT(!const_ || const_->addr >= word_count); - /* These structures are ordered by their address. */ - if (label && label->size == word_count) { - /* Just recording the address. */ - label->addr = (sljit_uw) code_ptr; - label->size = code_ptr - code; - label = label->next; - } - - if (jump && jump->addr == word_count) { - if (jump->flags & IS_JAL) - jump->addr = (sljit_uw)(code_ptr - 4); - else - jump->addr = (sljit_uw)(code_ptr - 3); - - code_ptr = detect_jump_type(jump, code_ptr, code); - jump = jump->next; - } - - if (const_ && const_->addr == word_count) { - /* Just recording the address. */ - const_->addr = (sljit_uw) code_ptr; - const_ = const_->next; - } - - code_ptr++; - word_count++; - } while (buf_ptr < buf_end); - - buf = buf->next; - } while (buf); - - if (label && label->size == word_count) { - label->addr = (sljit_uw) code_ptr; - label->size = code_ptr - code; - label = label->next; - } - - SLJIT_ASSERT(!label); - SLJIT_ASSERT(!jump); - SLJIT_ASSERT(!const_); - SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size); - - jump = compiler->jumps; - while (jump) { - do { - addr = (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target; - buf_ptr = (sljit_ins *)jump->addr; - - if (jump->flags & PATCH_B) { - addr = (sljit_sw)(addr - (jump->addr)) >> 3; - SLJIT_ASSERT((sljit_sw) addr <= SIMM_17BIT_MAX && (sljit_sw) addr >= SIMM_17BIT_MIN); - buf_ptr[0] = (buf_ptr[0] & ~(BOFF_X1(-1))) | BOFF_X1(addr); - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(buf_ptr); -#endif - break; - } - - if (jump->flags & PATCH_J) { - SLJIT_ASSERT((addr & ~0x3FFFFFFFL) == ((jump->addr + sizeof(sljit_ins)) & ~0x3FFFFFFFL)); - addr = (sljit_sw)(addr - (jump->addr)) >> 3; - buf_ptr[0] = (buf_ptr[0] & ~(JOFF_X1(-1))) | JOFF_X1(addr); - -#ifdef TILEGX_JIT_DEBUG - printf("[runtime relocate]%04d:\t", __LINE__); - print_insn_tilegx(buf_ptr); -#endif - break; - } - - SLJIT_ASSERT(!(jump->flags & IS_JAL)); - - /* Set the fields of immediate loads. */ - buf_ptr[0] = (buf_ptr[0] & ~(0xFFFFL << 43)) | (((addr >> 32) & 0xFFFFL) << 43); - buf_ptr[1] = (buf_ptr[1] & ~(0xFFFFL << 43)) | (((addr >> 16) & 0xFFFFL) << 43); - buf_ptr[2] = (buf_ptr[2] & ~(0xFFFFL << 43)) | ((addr & 0xFFFFL) << 43); - } while (0); - - jump = jump->next; - } - - compiler->error = SLJIT_ERR_COMPILED; - compiler->executable_size = (code_ptr - code) * sizeof(sljit_ins); - SLJIT_CACHE_FLUSH(code, code_ptr); - return code; -} - -static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 dst_ar, sljit_sw imm) -{ - - if (imm <= SIMM_16BIT_MAX && imm >= SIMM_16BIT_MIN) - return ADDLI(dst_ar, ZERO, imm); - - if (imm <= SIMM_32BIT_MAX && imm >= SIMM_32BIT_MIN) { - FAIL_IF(ADDLI(dst_ar, ZERO, imm >> 16)); - return SHL16INSLI(dst_ar, dst_ar, imm); - } - - if (imm <= SIMM_48BIT_MAX && imm >= SIMM_48BIT_MIN) { - FAIL_IF(ADDLI(dst_ar, ZERO, imm >> 32)); - FAIL_IF(SHL16INSLI(dst_ar, dst_ar, imm >> 16)); - return SHL16INSLI(dst_ar, dst_ar, imm); - } - - FAIL_IF(ADDLI(dst_ar, ZERO, imm >> 48)); - FAIL_IF(SHL16INSLI(dst_ar, dst_ar, imm >> 32)); - FAIL_IF(SHL16INSLI(dst_ar, dst_ar, imm >> 16)); - return SHL16INSLI(dst_ar, dst_ar, imm); -} - -static sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_s32 dst_ar, sljit_sw imm, int flush) -{ - /* Should *not* be optimized as load_immediate, as pcre relocation - mechanism will match this fixed 4-instruction pattern. */ - if (flush) { - FAIL_IF(ADDLI_SOLO(dst_ar, ZERO, imm >> 32)); - FAIL_IF(SHL16INSLI_SOLO(dst_ar, dst_ar, imm >> 16)); - return SHL16INSLI_SOLO(dst_ar, dst_ar, imm); - } - - FAIL_IF(ADDLI(dst_ar, ZERO, imm >> 32)); - FAIL_IF(SHL16INSLI(dst_ar, dst_ar, imm >> 16)); - return SHL16INSLI(dst_ar, dst_ar, imm); -} - -static sljit_s32 emit_const_64(struct sljit_compiler *compiler, sljit_s32 dst_ar, sljit_sw imm, int flush) -{ - /* Should *not* be optimized as load_immediate, as pcre relocation - mechanism will match this fixed 4-instruction pattern. */ - if (flush) { - FAIL_IF(ADDLI_SOLO(reg_map[dst_ar], ZERO, imm >> 48)); - FAIL_IF(SHL16INSLI_SOLO(reg_map[dst_ar], reg_map[dst_ar], imm >> 32)); - FAIL_IF(SHL16INSLI_SOLO(reg_map[dst_ar], reg_map[dst_ar], imm >> 16)); - return SHL16INSLI_SOLO(reg_map[dst_ar], reg_map[dst_ar], imm); - } - - FAIL_IF(ADDLI(reg_map[dst_ar], ZERO, imm >> 48)); - FAIL_IF(SHL16INSLI(reg_map[dst_ar], reg_map[dst_ar], imm >> 32)); - FAIL_IF(SHL16INSLI(reg_map[dst_ar], reg_map[dst_ar], imm >> 16)); - return SHL16INSLI(reg_map[dst_ar], reg_map[dst_ar], imm); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler, - sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds, - sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) -{ - sljit_ins base; - sljit_s32 i, tmp; - - CHECK_ERROR(); - CHECK(check_sljit_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size)); - set_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size); - - local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1); - local_size = (local_size + 7) & ~7; - compiler->local_size = local_size; - - if (local_size <= SIMM_16BIT_MAX) { - /* Frequent case. */ - FAIL_IF(ADDLI(SLJIT_LOCALS_REG_mapped, SLJIT_LOCALS_REG_mapped, -local_size)); - base = SLJIT_LOCALS_REG_mapped; - } else { - FAIL_IF(load_immediate(compiler, TMP_REG1_mapped, local_size)); - FAIL_IF(ADD(TMP_REG2_mapped, SLJIT_LOCALS_REG_mapped, ZERO)); - FAIL_IF(SUB(SLJIT_LOCALS_REG_mapped, SLJIT_LOCALS_REG_mapped, TMP_REG1_mapped)); - base = TMP_REG2_mapped; - local_size = 0; - } - - /* Save the return address. */ - FAIL_IF(ADDLI(ADDR_TMP_mapped, base, local_size - 8)); - FAIL_IF(ST_ADD(ADDR_TMP_mapped, RA, -8)); - - /* Save the S registers. */ - tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG; - for (i = SLJIT_S0; i >= tmp; i--) { - FAIL_IF(ST_ADD(ADDR_TMP_mapped, reg_map[i], -8)); - } - - /* Save the R registers that need to be reserved. */ - for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) { - FAIL_IF(ST_ADD(ADDR_TMP_mapped, reg_map[i], -8)); - } - - /* Move the arguments to S registers. */ - for (i = 0; i < args; i++) { - FAIL_IF(ADD(reg_map[SLJIT_S0 - i], i, ZERO)); - } - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler, - sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds, - sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) -{ - CHECK_ERROR(); - CHECK(check_sljit_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size)); - set_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size); - - local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1); - compiler->local_size = (local_size + 7) & ~7; - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw) -{ - sljit_s32 local_size; - sljit_ins base; - sljit_s32 i, tmp; - sljit_s32 saveds; - - CHECK_ERROR(); - CHECK(check_sljit_emit_return(compiler, op, src, srcw)); - - FAIL_IF(emit_mov_before_return(compiler, op, src, srcw)); - - local_size = compiler->local_size; - if (local_size <= SIMM_16BIT_MAX) - base = SLJIT_LOCALS_REG_mapped; - else { - FAIL_IF(load_immediate(compiler, TMP_REG1_mapped, local_size)); - FAIL_IF(ADD(TMP_REG1_mapped, SLJIT_LOCALS_REG_mapped, TMP_REG1_mapped)); - base = TMP_REG1_mapped; - local_size = 0; - } - - /* Restore the return address. */ - FAIL_IF(ADDLI(ADDR_TMP_mapped, base, local_size - 8)); - FAIL_IF(LD_ADD(RA, ADDR_TMP_mapped, -8)); - - /* Restore the S registers. */ - saveds = compiler->saveds; - tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG; - for (i = SLJIT_S0; i >= tmp; i--) { - FAIL_IF(LD_ADD(reg_map[i], ADDR_TMP_mapped, -8)); - } - - /* Restore the R registers that need to be reserved. */ - for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) { - FAIL_IF(LD_ADD(reg_map[i], ADDR_TMP_mapped, -8)); - } - - if (compiler->local_size <= SIMM_16BIT_MAX) - FAIL_IF(ADDLI(SLJIT_LOCALS_REG_mapped, SLJIT_LOCALS_REG_mapped, compiler->local_size)); - else - FAIL_IF(ADD(SLJIT_LOCALS_REG_mapped, TMP_REG1_mapped, ZERO)); - - return JR(RA); -} - -/* reg_ar is an absoulute register! */ - -/* Can perform an operation using at most 1 instruction. */ -static sljit_s32 getput_arg_fast(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg_ar, sljit_s32 arg, sljit_sw argw) -{ - SLJIT_ASSERT(arg & SLJIT_MEM); - - if ((!(flags & WRITE_BACK) || !(arg & REG_MASK)) - && !(arg & OFFS_REG_MASK) && argw <= SIMM_16BIT_MAX && argw >= SIMM_16BIT_MIN) { - /* Works for both absoulte and relative addresses. */ - if (SLJIT_UNLIKELY(flags & ARG_TEST)) - return 1; - - FAIL_IF(ADDLI(ADDR_TMP_mapped, reg_map[arg & REG_MASK], argw)); - - if (flags & LOAD_DATA) - FAIL_IF(PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, ADDR_TMP_mapped)); - else - FAIL_IF(PB2(data_transfer_insts[flags & MEM_MASK], ADDR_TMP_mapped, reg_ar)); - - return -1; - } - - return 0; -} - -/* See getput_arg below. - Note: can_cache is called only for binary operators. Those - operators always uses word arguments without write back. */ -static sljit_s32 can_cache(sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw) -{ - SLJIT_ASSERT((arg & SLJIT_MEM) && (next_arg & SLJIT_MEM)); - - /* Simple operation except for updates. */ - if (arg & OFFS_REG_MASK) { - argw &= 0x3; - next_argw &= 0x3; - if (argw && argw == next_argw - && (arg == next_arg || (arg & OFFS_REG_MASK) == (next_arg & OFFS_REG_MASK))) - return 1; - return 0; - } - - if (arg == next_arg) { - if (((next_argw - argw) <= SIMM_16BIT_MAX - && (next_argw - argw) >= SIMM_16BIT_MIN)) - return 1; - - return 0; - } - - return 0; -} - -/* Emit the necessary instructions. See can_cache above. */ -static sljit_s32 getput_arg(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg_ar, sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw) -{ - sljit_s32 tmp_ar, base; - - SLJIT_ASSERT(arg & SLJIT_MEM); - if (!(next_arg & SLJIT_MEM)) { - next_arg = 0; - next_argw = 0; - } - - if ((flags & MEM_MASK) <= GPR_REG && (flags & LOAD_DATA)) - tmp_ar = reg_ar; - else - tmp_ar = TMP_REG1_mapped; - - base = arg & REG_MASK; - - if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) { - argw &= 0x3; - - if ((flags & WRITE_BACK) && reg_ar == reg_map[base]) { - SLJIT_ASSERT(!(flags & LOAD_DATA) && reg_map[TMP_REG1] != reg_ar); - FAIL_IF(ADD(TMP_REG1_mapped, reg_ar, ZERO)); - reg_ar = TMP_REG1_mapped; - } - - /* Using the cache. */ - if (argw == compiler->cache_argw) { - if (!(flags & WRITE_BACK)) { - if (arg == compiler->cache_arg) { - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, TMP_REG3_mapped); - else - return PB2(data_transfer_insts[flags & MEM_MASK], TMP_REG3_mapped, reg_ar); - } - - if ((SLJIT_MEM | (arg & OFFS_REG_MASK)) == compiler->cache_arg) { - if (arg == next_arg && argw == (next_argw & 0x3)) { - compiler->cache_arg = arg; - compiler->cache_argw = argw; - FAIL_IF(ADD(TMP_REG3_mapped, reg_map[base], TMP_REG3_mapped)); - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, TMP_REG3_mapped); - else - return PB2(data_transfer_insts[flags & MEM_MASK], TMP_REG3_mapped, reg_ar); - } - - FAIL_IF(ADD(tmp_ar, reg_map[base], TMP_REG3_mapped)); - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, tmp_ar); - else - return PB2(data_transfer_insts[flags & MEM_MASK], tmp_ar, reg_ar); - } - } else { - if ((SLJIT_MEM | (arg & OFFS_REG_MASK)) == compiler->cache_arg) { - FAIL_IF(ADD(reg_map[base], reg_map[base], TMP_REG3_mapped)); - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, reg_map[base]); - else - return PB2(data_transfer_insts[flags & MEM_MASK], reg_map[base], reg_ar); - } - } - } - - if (SLJIT_UNLIKELY(argw)) { - compiler->cache_arg = SLJIT_MEM | (arg & OFFS_REG_MASK); - compiler->cache_argw = argw; - FAIL_IF(SHLI(TMP_REG3_mapped, reg_map[OFFS_REG(arg)], argw)); - } - - if (!(flags & WRITE_BACK)) { - if (arg == next_arg && argw == (next_argw & 0x3)) { - compiler->cache_arg = arg; - compiler->cache_argw = argw; - FAIL_IF(ADD(TMP_REG3_mapped, reg_map[base], reg_map[!argw ? OFFS_REG(arg) : TMP_REG3])); - tmp_ar = TMP_REG3_mapped; - } else - FAIL_IF(ADD(tmp_ar, reg_map[base], reg_map[!argw ? OFFS_REG(arg) : TMP_REG3])); - - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, tmp_ar); - else - return PB2(data_transfer_insts[flags & MEM_MASK], tmp_ar, reg_ar); - } - - FAIL_IF(ADD(reg_map[base], reg_map[base], reg_map[!argw ? OFFS_REG(arg) : TMP_REG3])); - - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, reg_map[base]); - else - return PB2(data_transfer_insts[flags & MEM_MASK], reg_map[base], reg_ar); - } - - if (SLJIT_UNLIKELY(flags & WRITE_BACK) && base) { - /* Update only applies if a base register exists. */ - if (reg_ar == reg_map[base]) { - SLJIT_ASSERT(!(flags & LOAD_DATA) && TMP_REG1_mapped != reg_ar); - if (argw <= SIMM_16BIT_MAX && argw >= SIMM_16BIT_MIN) { - FAIL_IF(ADDLI(ADDR_TMP_mapped, reg_map[base], argw)); - if (flags & LOAD_DATA) - FAIL_IF(PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, ADDR_TMP_mapped)); - else - FAIL_IF(PB2(data_transfer_insts[flags & MEM_MASK], ADDR_TMP_mapped, reg_ar)); - - if (argw) - return ADDLI(reg_map[base], reg_map[base], argw); - - return SLJIT_SUCCESS; - } - - FAIL_IF(ADD(TMP_REG1_mapped, reg_ar, ZERO)); - reg_ar = TMP_REG1_mapped; - } - - if (argw <= SIMM_16BIT_MAX && argw >= SIMM_16BIT_MIN) { - if (argw) - FAIL_IF(ADDLI(reg_map[base], reg_map[base], argw)); - } else { - if (compiler->cache_arg == SLJIT_MEM - && argw - compiler->cache_argw <= SIMM_16BIT_MAX - && argw - compiler->cache_argw >= SIMM_16BIT_MIN) { - if (argw != compiler->cache_argw) { - FAIL_IF(ADD(TMP_REG3_mapped, TMP_REG3_mapped, argw - compiler->cache_argw)); - compiler->cache_argw = argw; - } - - FAIL_IF(ADD(reg_map[base], reg_map[base], TMP_REG3_mapped)); - } else { - compiler->cache_arg = SLJIT_MEM; - compiler->cache_argw = argw; - FAIL_IF(load_immediate(compiler, TMP_REG3_mapped, argw)); - FAIL_IF(ADD(reg_map[base], reg_map[base], TMP_REG3_mapped)); - } - } - - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, reg_map[base]); - else - return PB2(data_transfer_insts[flags & MEM_MASK], reg_map[base], reg_ar); - } - - if (compiler->cache_arg == arg - && argw - compiler->cache_argw <= SIMM_16BIT_MAX - && argw - compiler->cache_argw >= SIMM_16BIT_MIN) { - if (argw != compiler->cache_argw) { - FAIL_IF(ADDLI(TMP_REG3_mapped, TMP_REG3_mapped, argw - compiler->cache_argw)); - compiler->cache_argw = argw; - } - - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, TMP_REG3_mapped); - else - return PB2(data_transfer_insts[flags & MEM_MASK], TMP_REG3_mapped, reg_ar); - } - - if (compiler->cache_arg == SLJIT_MEM - && argw - compiler->cache_argw <= SIMM_16BIT_MAX - && argw - compiler->cache_argw >= SIMM_16BIT_MIN) { - if (argw != compiler->cache_argw) - FAIL_IF(ADDLI(TMP_REG3_mapped, TMP_REG3_mapped, argw - compiler->cache_argw)); - } else { - compiler->cache_arg = SLJIT_MEM; - FAIL_IF(load_immediate(compiler, TMP_REG3_mapped, argw)); - } - - compiler->cache_argw = argw; - - if (!base) { - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, TMP_REG3_mapped); - else - return PB2(data_transfer_insts[flags & MEM_MASK], TMP_REG3_mapped, reg_ar); - } - - if (arg == next_arg - && next_argw - argw <= SIMM_16BIT_MAX - && next_argw - argw >= SIMM_16BIT_MIN) { - compiler->cache_arg = arg; - FAIL_IF(ADD(TMP_REG3_mapped, TMP_REG3_mapped, reg_map[base])); - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, TMP_REG3_mapped); - else - return PB2(data_transfer_insts[flags & MEM_MASK], TMP_REG3_mapped, reg_ar); - } - - FAIL_IF(ADD(tmp_ar, TMP_REG3_mapped, reg_map[base])); - - if (flags & LOAD_DATA) - return PB2(data_transfer_insts[flags & MEM_MASK], reg_ar, tmp_ar); - else - return PB2(data_transfer_insts[flags & MEM_MASK], tmp_ar, reg_ar); -} - -static SLJIT_INLINE sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg_ar, sljit_s32 arg, sljit_sw argw) -{ - if (getput_arg_fast(compiler, flags, reg_ar, arg, argw)) - return compiler->error; - - compiler->cache_arg = 0; - compiler->cache_argw = 0; - return getput_arg(compiler, flags, reg_ar, arg, argw, 0, 0); -} - -static SLJIT_INLINE sljit_s32 emit_op_mem2(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg1, sljit_sw arg1w, sljit_s32 arg2, sljit_sw arg2w) -{ - if (getput_arg_fast(compiler, flags, reg, arg1, arg1w)) - return compiler->error; - return getput_arg(compiler, flags, reg, arg1, arg1w, arg2, arg2w); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw)); - ADJUST_LOCAL_OFFSET(dst, dstw); - - /* For UNUSED dst. Uncommon, but possible. */ - if (dst == SLJIT_UNUSED) - return SLJIT_SUCCESS; - - if (FAST_IS_REG(dst)) - return ADD(reg_map[dst], RA, ZERO); - - /* Memory. */ - return emit_op_mem(compiler, WORD_DATA, RA, dst, dstw); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) - FAIL_IF(ADD(RA, reg_map[src], ZERO)); - - else if (src & SLJIT_MEM) - FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, RA, src, srcw)); - - else if (src & SLJIT_IMM) - FAIL_IF(load_immediate(compiler, RA, srcw)); - - return JR(RA); -} - -static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags, sljit_s32 dst, sljit_s32 src1, sljit_sw src2) -{ - sljit_s32 overflow_ra = 0; - - switch (GET_OPCODE(op)) { - case SLJIT_MOV: - case SLJIT_MOV_P: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if (dst != src2) - return ADD(reg_map[dst], reg_map[src2], ZERO); - return SLJIT_SUCCESS; - - case SLJIT_MOV_U32: - case SLJIT_MOV_S32: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) { - if (op == SLJIT_MOV_S32) - return BFEXTS(reg_map[dst], reg_map[src2], 0, 31); - - return BFEXTU(reg_map[dst], reg_map[src2], 0, 31); - } else if (dst != src2) { - SLJIT_ASSERT(src2 == 0); - return ADD(reg_map[dst], reg_map[src2], ZERO); - } - - return SLJIT_SUCCESS; - - case SLJIT_MOV_U8: - case SLJIT_MOV_S8: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) { - if (op == SLJIT_MOV_S8) - return BFEXTS(reg_map[dst], reg_map[src2], 0, 7); - - return BFEXTU(reg_map[dst], reg_map[src2], 0, 7); - } else if (dst != src2) { - SLJIT_ASSERT(src2 == 0); - return ADD(reg_map[dst], reg_map[src2], ZERO); - } - - return SLJIT_SUCCESS; - - case SLJIT_MOV_U16: - case SLJIT_MOV_S16: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) { - if (op == SLJIT_MOV_S16) - return BFEXTS(reg_map[dst], reg_map[src2], 0, 15); - - return BFEXTU(reg_map[dst], reg_map[src2], 0, 15); - } else if (dst != src2) { - SLJIT_ASSERT(src2 == 0); - return ADD(reg_map[dst], reg_map[src2], ZERO); - } - - return SLJIT_SUCCESS; - - case SLJIT_NOT: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if (op & SLJIT_SET_E) - FAIL_IF(NOR(EQUAL_FLAG, reg_map[src2], reg_map[src2])); - if (CHECK_FLAGS(SLJIT_SET_E)) - FAIL_IF(NOR(reg_map[dst], reg_map[src2], reg_map[src2])); - - return SLJIT_SUCCESS; - - case SLJIT_CLZ: - SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM)); - if (op & SLJIT_SET_E) - FAIL_IF(CLZ(EQUAL_FLAG, reg_map[src2])); - if (CHECK_FLAGS(SLJIT_SET_E)) - FAIL_IF(CLZ(reg_map[dst], reg_map[src2])); - - return SLJIT_SUCCESS; - - case SLJIT_ADD: - if (flags & SRC2_IMM) { - if (op & SLJIT_SET_O) { - FAIL_IF(SHRUI(TMP_EREG1, reg_map[src1], 63)); - if (src2 < 0) - FAIL_IF(XORI(TMP_EREG1, TMP_EREG1, 1)); - } - - if (op & SLJIT_SET_E) - FAIL_IF(ADDLI(EQUAL_FLAG, reg_map[src1], src2)); - - if (op & SLJIT_SET_C) { - if (src2 >= 0) - FAIL_IF(ORI(ULESS_FLAG ,reg_map[src1], src2)); - else { - FAIL_IF(ADDLI(ULESS_FLAG ,ZERO, src2)); - FAIL_IF(OR(ULESS_FLAG,reg_map[src1],ULESS_FLAG)); - } - } - - /* dst may be the same as src1 or src2. */ - if (CHECK_FLAGS(SLJIT_SET_E)) - FAIL_IF(ADDLI(reg_map[dst], reg_map[src1], src2)); - - if (op & SLJIT_SET_O) { - FAIL_IF(SHRUI(OVERFLOW_FLAG, reg_map[dst], 63)); - - if (src2 < 0) - FAIL_IF(XORI(OVERFLOW_FLAG, OVERFLOW_FLAG, 1)); - } - } else { - if (op & SLJIT_SET_O) { - FAIL_IF(XOR(TMP_EREG1, reg_map[src1], reg_map[src2])); - FAIL_IF(SHRUI(TMP_EREG1, TMP_EREG1, 63)); - - if (src1 != dst) - overflow_ra = reg_map[src1]; - else if (src2 != dst) - overflow_ra = reg_map[src2]; - else { - /* Rare ocasion. */ - FAIL_IF(ADD(TMP_EREG2, reg_map[src1], ZERO)); - overflow_ra = TMP_EREG2; - } - } - - if (op & SLJIT_SET_E) - FAIL_IF(ADD(EQUAL_FLAG ,reg_map[src1], reg_map[src2])); - - if (op & SLJIT_SET_C) - FAIL_IF(OR(ULESS_FLAG,reg_map[src1], reg_map[src2])); - - /* dst may be the same as src1 or src2. */ - if (CHECK_FLAGS(SLJIT_SET_E)) - FAIL_IF(ADD(reg_map[dst],reg_map[src1], reg_map[src2])); - - if (op & SLJIT_SET_O) { - FAIL_IF(XOR(OVERFLOW_FLAG,reg_map[dst], overflow_ra)); - FAIL_IF(SHRUI(OVERFLOW_FLAG, OVERFLOW_FLAG, 63)); - } - } - - /* a + b >= a | b (otherwise, the carry should be set to 1). */ - if (op & SLJIT_SET_C) - FAIL_IF(CMPLTU(ULESS_FLAG ,reg_map[dst] ,ULESS_FLAG)); - - if (op & SLJIT_SET_O) - return CMOVNEZ(OVERFLOW_FLAG, TMP_EREG1, ZERO); - - return SLJIT_SUCCESS; - - case SLJIT_ADDC: - if (flags & SRC2_IMM) { - if (op & SLJIT_SET_C) { - if (src2 >= 0) - FAIL_IF(ORI(TMP_EREG1, reg_map[src1], src2)); - else { - FAIL_IF(ADDLI(TMP_EREG1, ZERO, src2)); - FAIL_IF(OR(TMP_EREG1, reg_map[src1], TMP_EREG1)); - } - } - - FAIL_IF(ADDLI(reg_map[dst], reg_map[src1], src2)); - - } else { - if (op & SLJIT_SET_C) - FAIL_IF(OR(TMP_EREG1, reg_map[src1], reg_map[src2])); - - /* dst may be the same as src1 or src2. */ - FAIL_IF(ADD(reg_map[dst], reg_map[src1], reg_map[src2])); - } - - if (op & SLJIT_SET_C) - FAIL_IF(CMPLTU(TMP_EREG1, reg_map[dst], TMP_EREG1)); - - FAIL_IF(ADD(reg_map[dst], reg_map[dst], ULESS_FLAG)); - - if (!(op & SLJIT_SET_C)) - return SLJIT_SUCCESS; - - /* Set TMP_EREG2 (dst == 0) && (ULESS_FLAG == 1). */ - FAIL_IF(CMPLTUI(TMP_EREG2, reg_map[dst], 1)); - FAIL_IF(AND(TMP_EREG2, TMP_EREG2, ULESS_FLAG)); - /* Set carry flag. */ - return OR(ULESS_FLAG, TMP_EREG2, TMP_EREG1); - - case SLJIT_SUB: - if ((flags & SRC2_IMM) && ((op & (SLJIT_SET_U | SLJIT_SET_S)) || src2 == SIMM_16BIT_MIN)) { - FAIL_IF(ADDLI(TMP_REG2_mapped, ZERO, src2)); - src2 = TMP_REG2; - flags &= ~SRC2_IMM; - } - - if (flags & SRC2_IMM) { - if (op & SLJIT_SET_O) { - FAIL_IF(SHRUI(TMP_EREG1,reg_map[src1], 63)); - - if (src2 < 0) - FAIL_IF(XORI(TMP_EREG1, TMP_EREG1, 1)); - - if (src1 != dst) - overflow_ra = reg_map[src1]; - else { - /* Rare ocasion. */ - FAIL_IF(ADD(TMP_EREG2, reg_map[src1], ZERO)); - overflow_ra = TMP_EREG2; - } - } - - if (op & SLJIT_SET_E) - FAIL_IF(ADDLI(EQUAL_FLAG, reg_map[src1], -src2)); - - if (op & SLJIT_SET_C) { - FAIL_IF(load_immediate(compiler, ADDR_TMP_mapped, src2)); - FAIL_IF(CMPLTU(ULESS_FLAG, reg_map[src1], ADDR_TMP_mapped)); - } - - /* dst may be the same as src1 or src2. */ - if (CHECK_FLAGS(SLJIT_SET_E)) - FAIL_IF(ADDLI(reg_map[dst], reg_map[src1], -src2)); - - } else { - - if (op & SLJIT_SET_O) { - FAIL_IF(XOR(TMP_EREG1, reg_map[src1], reg_map[src2])); - FAIL_IF(SHRUI(TMP_EREG1, TMP_EREG1, 63)); - - if (src1 != dst) - overflow_ra = reg_map[src1]; - else { - /* Rare ocasion. */ - FAIL_IF(ADD(TMP_EREG2, reg_map[src1], ZERO)); - overflow_ra = TMP_EREG2; - } - } - - if (op & SLJIT_SET_E) - FAIL_IF(SUB(EQUAL_FLAG, reg_map[src1], reg_map[src2])); - - if (op & (SLJIT_SET_U | SLJIT_SET_C)) - FAIL_IF(CMPLTU(ULESS_FLAG, reg_map[src1], reg_map[src2])); - - if (op & SLJIT_SET_U) - FAIL_IF(CMPLTU(UGREATER_FLAG, reg_map[src2], reg_map[src1])); - - if (op & SLJIT_SET_S) { - FAIL_IF(CMPLTS(LESS_FLAG ,reg_map[src1] ,reg_map[src2])); - FAIL_IF(CMPLTS(GREATER_FLAG ,reg_map[src2] ,reg_map[src1])); - } - - /* dst may be the same as src1 or src2. */ - if (CHECK_FLAGS(SLJIT_SET_E | SLJIT_SET_U | SLJIT_SET_S | SLJIT_SET_C)) - FAIL_IF(SUB(reg_map[dst], reg_map[src1], reg_map[src2])); - } - - if (op & SLJIT_SET_O) { - FAIL_IF(XOR(OVERFLOW_FLAG, reg_map[dst], overflow_ra)); - FAIL_IF(SHRUI(OVERFLOW_FLAG, OVERFLOW_FLAG, 63)); - return CMOVEQZ(OVERFLOW_FLAG, TMP_EREG1, ZERO); - } - - return SLJIT_SUCCESS; - - case SLJIT_SUBC: - if ((flags & SRC2_IMM) && src2 == SIMM_16BIT_MIN) { - FAIL_IF(ADDLI(TMP_REG2_mapped, ZERO, src2)); - src2 = TMP_REG2; - flags &= ~SRC2_IMM; - } - - if (flags & SRC2_IMM) { - if (op & SLJIT_SET_C) { - FAIL_IF(load_immediate(compiler, ADDR_TMP_mapped, -src2)); - FAIL_IF(CMPLTU(TMP_EREG1, reg_map[src1], ADDR_TMP_mapped)); - } - - /* dst may be the same as src1 or src2. */ - FAIL_IF(ADDLI(reg_map[dst], reg_map[src1], -src2)); - - } else { - if (op & SLJIT_SET_C) - FAIL_IF(CMPLTU(TMP_EREG1, reg_map[src1], reg_map[src2])); - /* dst may be the same as src1 or src2. */ - FAIL_IF(SUB(reg_map[dst], reg_map[src1], reg_map[src2])); - } - - if (op & SLJIT_SET_C) - FAIL_IF(CMOVEQZ(TMP_EREG1, reg_map[dst], ULESS_FLAG)); - - FAIL_IF(SUB(reg_map[dst], reg_map[dst], ULESS_FLAG)); - - if (op & SLJIT_SET_C) - FAIL_IF(ADD(ULESS_FLAG, TMP_EREG1, ZERO)); - - return SLJIT_SUCCESS; - - case SLJIT_MUL: - if (flags & SRC2_IMM) { - FAIL_IF(load_immediate(compiler, TMP_REG2_mapped, src2)); - src2 = TMP_REG2; - flags &= ~SRC2_IMM; - } - - FAIL_IF(MUL(reg_map[dst], reg_map[src1], reg_map[src2])); - - return SLJIT_SUCCESS; - -#define EMIT_LOGICAL(op_imm, op_norm) \ - if (flags & SRC2_IMM) { \ - FAIL_IF(load_immediate(compiler, ADDR_TMP_mapped, src2)); \ - if (op & SLJIT_SET_E) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, EQUAL_FLAG, reg_map[src1], \ - ADDR_TMP_mapped, __LINE__)); \ - if (CHECK_FLAGS(SLJIT_SET_E)) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, reg_map[dst], reg_map[src1], \ - ADDR_TMP_mapped, __LINE__)); \ - } else { \ - if (op & SLJIT_SET_E) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, EQUAL_FLAG, reg_map[src1], \ - reg_map[src2], __LINE__)); \ - if (CHECK_FLAGS(SLJIT_SET_E)) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, reg_map[dst], reg_map[src1], \ - reg_map[src2], __LINE__)); \ - } - - case SLJIT_AND: - EMIT_LOGICAL(TILEGX_OPC_ANDI, TILEGX_OPC_AND); - return SLJIT_SUCCESS; - - case SLJIT_OR: - EMIT_LOGICAL(TILEGX_OPC_ORI, TILEGX_OPC_OR); - return SLJIT_SUCCESS; - - case SLJIT_XOR: - EMIT_LOGICAL(TILEGX_OPC_XORI, TILEGX_OPC_XOR); - return SLJIT_SUCCESS; - -#define EMIT_SHIFT(op_imm, op_norm) \ - if (flags & SRC2_IMM) { \ - if (op & SLJIT_SET_E) \ - FAIL_IF(push_3_buffer( \ - compiler, op_imm, EQUAL_FLAG, reg_map[src1], \ - src2 & 0x3F, __LINE__)); \ - if (CHECK_FLAGS(SLJIT_SET_E)) \ - FAIL_IF(push_3_buffer( \ - compiler, op_imm, reg_map[dst], reg_map[src1], \ - src2 & 0x3F, __LINE__)); \ - } else { \ - if (op & SLJIT_SET_E) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, EQUAL_FLAG, reg_map[src1], \ - reg_map[src2], __LINE__)); \ - if (CHECK_FLAGS(SLJIT_SET_E)) \ - FAIL_IF(push_3_buffer( \ - compiler, op_norm, reg_map[dst], reg_map[src1], \ - reg_map[src2], __LINE__)); \ - } - - case SLJIT_SHL: - EMIT_SHIFT(TILEGX_OPC_SHLI, TILEGX_OPC_SHL); - return SLJIT_SUCCESS; - - case SLJIT_LSHR: - EMIT_SHIFT(TILEGX_OPC_SHRUI, TILEGX_OPC_SHRU); - return SLJIT_SUCCESS; - - case SLJIT_ASHR: - EMIT_SHIFT(TILEGX_OPC_SHRSI, TILEGX_OPC_SHRS); - return SLJIT_SUCCESS; - } - - SLJIT_UNREACHABLE(); - return SLJIT_SUCCESS; -} - -static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags, sljit_s32 dst, sljit_sw dstw, sljit_s32 src1, sljit_sw src1w, sljit_s32 src2, sljit_sw src2w) -{ - /* arg1 goes to TMP_REG1 or src reg. - arg2 goes to TMP_REG2, imm or src reg. - TMP_REG3 can be used for caching. - result goes to TMP_REG2, so put result can use TMP_REG1 and TMP_REG3. */ - sljit_s32 dst_r = TMP_REG2; - sljit_s32 src1_r; - sljit_sw src2_r = 0; - sljit_s32 sugg_src2_r = TMP_REG2; - - if (!(flags & ALT_KEEP_CACHE)) { - compiler->cache_arg = 0; - compiler->cache_argw = 0; - } - - if (SLJIT_UNLIKELY(dst == SLJIT_UNUSED)) { - if (op >= SLJIT_MOV && op <= SLJIT_MOVU_S32 && !(src2 & SLJIT_MEM)) - return SLJIT_SUCCESS; - if (GET_FLAGS(op)) - flags |= UNUSED_DEST; - } else if (FAST_IS_REG(dst)) { - dst_r = dst; - flags |= REG_DEST; - if (op >= SLJIT_MOV && op <= SLJIT_MOVU_S32) - sugg_src2_r = dst_r; - } else if ((dst & SLJIT_MEM) && !getput_arg_fast(compiler, flags | ARG_TEST, TMP_REG1_mapped, dst, dstw)) - flags |= SLOW_DEST; - - if (flags & IMM_OP) { - if ((src2 & SLJIT_IMM) && src2w) { - if ((!(flags & LOGICAL_OP) - && (src2w <= SIMM_16BIT_MAX && src2w >= SIMM_16BIT_MIN)) - || ((flags & LOGICAL_OP) && !(src2w & ~UIMM_16BIT_MAX))) { - flags |= SRC2_IMM; - src2_r = src2w; - } - } - - if (!(flags & SRC2_IMM) && (flags & CUMULATIVE_OP) && (src1 & SLJIT_IMM) && src1w) { - if ((!(flags & LOGICAL_OP) - && (src1w <= SIMM_16BIT_MAX && src1w >= SIMM_16BIT_MIN)) - || ((flags & LOGICAL_OP) && !(src1w & ~UIMM_16BIT_MAX))) { - flags |= SRC2_IMM; - src2_r = src1w; - - /* And swap arguments. */ - src1 = src2; - src1w = src2w; - src2 = SLJIT_IMM; - /* src2w = src2_r unneeded. */ - } - } - } - - /* Source 1. */ - if (FAST_IS_REG(src1)) { - src1_r = src1; - flags |= REG1_SOURCE; - } else if (src1 & SLJIT_IMM) { - if (src1w) { - FAIL_IF(load_immediate(compiler, TMP_REG1_mapped, src1w)); - src1_r = TMP_REG1; - } else - src1_r = 0; - } else { - if (getput_arg_fast(compiler, flags | LOAD_DATA, TMP_REG1_mapped, src1, src1w)) - FAIL_IF(compiler->error); - else - flags |= SLOW_SRC1; - src1_r = TMP_REG1; - } - - /* Source 2. */ - if (FAST_IS_REG(src2)) { - src2_r = src2; - flags |= REG2_SOURCE; - if (!(flags & REG_DEST) && op >= SLJIT_MOV && op <= SLJIT_MOVU_S32) - dst_r = src2_r; - } else if (src2 & SLJIT_IMM) { - if (!(flags & SRC2_IMM)) { - if (src2w) { - FAIL_IF(load_immediate(compiler, reg_map[sugg_src2_r], src2w)); - src2_r = sugg_src2_r; - } else { - src2_r = 0; - if ((op >= SLJIT_MOV && op <= SLJIT_MOVU_S32) && (dst & SLJIT_MEM)) - dst_r = 0; - } - } - } else { - if (getput_arg_fast(compiler, flags | LOAD_DATA, reg_map[sugg_src2_r], src2, src2w)) - FAIL_IF(compiler->error); - else - flags |= SLOW_SRC2; - src2_r = sugg_src2_r; - } - - if ((flags & (SLOW_SRC1 | SLOW_SRC2)) == (SLOW_SRC1 | SLOW_SRC2)) { - SLJIT_ASSERT(src2_r == TMP_REG2); - if (!can_cache(src1, src1w, src2, src2w) && can_cache(src1, src1w, dst, dstw)) { - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG2_mapped, src2, src2w, src1, src1w)); - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1_mapped, src1, src1w, dst, dstw)); - } else { - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1_mapped, src1, src1w, src2, src2w)); - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG2_mapped, src2, src2w, dst, dstw)); - } - } else if (flags & SLOW_SRC1) - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1_mapped, src1, src1w, dst, dstw)); - else if (flags & SLOW_SRC2) - FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, reg_map[sugg_src2_r], src2, src2w, dst, dstw)); - - FAIL_IF(emit_single_op(compiler, op, flags, dst_r, src1_r, src2_r)); - - if (dst & SLJIT_MEM) { - if (!(flags & SLOW_DEST)) { - getput_arg_fast(compiler, flags, reg_map[dst_r], dst, dstw); - return compiler->error; - } - - return getput_arg(compiler, flags, reg_map[dst_r], dst, dstw, 0, 0); - } - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, sljit_s32 src, sljit_sw srcw, sljit_s32 type) -{ - sljit_s32 sugg_dst_ar, dst_ar; - sljit_s32 flags = GET_ALL_FLAGS(op); - sljit_s32 mem_type = (op & SLJIT_I32_OP) ? (INT_DATA | SIGNED_DATA) : WORD_DATA; - - CHECK_ERROR(); - CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, src, srcw, type)); - ADJUST_LOCAL_OFFSET(dst, dstw); - - op = GET_OPCODE(op); - if (op == SLJIT_MOV_S32 || op == SLJIT_MOV_U32) - mem_type = INT_DATA | SIGNED_DATA; - sugg_dst_ar = reg_map[(op < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG2]; - - compiler->cache_arg = 0; - compiler->cache_argw = 0; - if (op >= SLJIT_ADD && (src & SLJIT_MEM)) { - ADJUST_LOCAL_OFFSET(src, srcw); - FAIL_IF(emit_op_mem2(compiler, mem_type | LOAD_DATA, TMP_REG1_mapped, src, srcw, dst, dstw)); - src = TMP_REG1; - srcw = 0; - } - - switch (type & 0xff) { - case SLJIT_EQUAL: - case SLJIT_NOT_EQUAL: - FAIL_IF(CMPLTUI(sugg_dst_ar, EQUAL_FLAG, 1)); - dst_ar = sugg_dst_ar; - break; - case SLJIT_LESS: - case SLJIT_GREATER_EQUAL: - dst_ar = ULESS_FLAG; - break; - case SLJIT_GREATER: - case SLJIT_LESS_EQUAL: - dst_ar = UGREATER_FLAG; - break; - case SLJIT_SIG_LESS: - case SLJIT_SIG_GREATER_EQUAL: - dst_ar = LESS_FLAG; - break; - case SLJIT_SIG_GREATER: - case SLJIT_SIG_LESS_EQUAL: - dst_ar = GREATER_FLAG; - break; - case SLJIT_OVERFLOW: - case SLJIT_NOT_OVERFLOW: - dst_ar = OVERFLOW_FLAG; - break; - case SLJIT_MUL_OVERFLOW: - case SLJIT_MUL_NOT_OVERFLOW: - FAIL_IF(CMPLTUI(sugg_dst_ar, OVERFLOW_FLAG, 1)); - dst_ar = sugg_dst_ar; - type ^= 0x1; /* Flip type bit for the XORI below. */ - break; - - default: - SLJIT_UNREACHABLE(); - dst_ar = sugg_dst_ar; - break; - } - - if (type & 0x1) { - FAIL_IF(XORI(sugg_dst_ar, dst_ar, 1)); - dst_ar = sugg_dst_ar; - } - - if (op >= SLJIT_ADD) { - if (TMP_REG2_mapped != dst_ar) - FAIL_IF(ADD(TMP_REG2_mapped, dst_ar, ZERO)); - return emit_op(compiler, op | flags, mem_type | CUMULATIVE_OP | LOGICAL_OP | IMM_OP | ALT_KEEP_CACHE, dst, dstw, src, srcw, TMP_REG2, 0); - } - - if (dst & SLJIT_MEM) - return emit_op_mem(compiler, mem_type, dst_ar, dst, dstw); - - if (sugg_dst_ar != dst_ar) - return ADD(sugg_dst_ar, dst_ar, ZERO); - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op) { - CHECK_ERROR(); - CHECK(check_sljit_emit_op0(compiler, op)); - - op = GET_OPCODE(op); - switch (op) { - case SLJIT_NOP: - return push_0_buffer(compiler, TILEGX_OPC_FNOP, __LINE__); - - case SLJIT_BREAKPOINT: - return PI(BPT); - - case SLJIT_LMUL_UW: - case SLJIT_LMUL_SW: - case SLJIT_DIVMOD_UW: - case SLJIT_DIVMOD_SW: - case SLJIT_DIV_UW: - case SLJIT_DIV_SW: - SLJIT_UNREACHABLE(); - } - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, sljit_s32 src, sljit_sw srcw) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw)); - ADJUST_LOCAL_OFFSET(dst, dstw); - ADJUST_LOCAL_OFFSET(src, srcw); - - switch (GET_OPCODE(op)) { - case SLJIT_MOV: - case SLJIT_MOV_P: - return emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOV_U32: - return emit_op(compiler, SLJIT_MOV_U32, INT_DATA, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOV_S32: - return emit_op(compiler, SLJIT_MOV_S32, INT_DATA | SIGNED_DATA, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOV_U8: - return emit_op(compiler, SLJIT_MOV_U8, BYTE_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u8) srcw : srcw); - - case SLJIT_MOV_S8: - return emit_op(compiler, SLJIT_MOV_S8, BYTE_DATA | SIGNED_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s8) srcw : srcw); - - case SLJIT_MOV_U16: - return emit_op(compiler, SLJIT_MOV_U16, HALF_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u16) srcw : srcw); - - case SLJIT_MOV_S16: - return emit_op(compiler, SLJIT_MOV_S16, HALF_DATA | SIGNED_DATA, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s16) srcw : srcw); - - case SLJIT_MOVU: - case SLJIT_MOVU_P: - return emit_op(compiler, SLJIT_MOV, WORD_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOVU_U32: - return emit_op(compiler, SLJIT_MOV_U32, INT_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOVU_S32: - return emit_op(compiler, SLJIT_MOV_S32, INT_DATA | SIGNED_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_MOVU_U8: - return emit_op(compiler, SLJIT_MOV_U8, BYTE_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u8) srcw : srcw); - - case SLJIT_MOVU_S8: - return emit_op(compiler, SLJIT_MOV_S8, BYTE_DATA | SIGNED_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s8) srcw : srcw); - - case SLJIT_MOVU_U16: - return emit_op(compiler, SLJIT_MOV_U16, HALF_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_u16) srcw : srcw); - - case SLJIT_MOVU_S16: - return emit_op(compiler, SLJIT_MOV_S16, HALF_DATA | SIGNED_DATA | WRITE_BACK, dst, dstw, TMP_REG1, 0, src, (src & SLJIT_IMM) ? (sljit_s16) srcw : srcw); - - case SLJIT_NOT: - return emit_op(compiler, op, 0, dst, dstw, TMP_REG1, 0, src, srcw); - - case SLJIT_NEG: - return emit_op(compiler, SLJIT_SUB | GET_ALL_FLAGS(op), IMM_OP, dst, dstw, SLJIT_IMM, 0, src, srcw); - - case SLJIT_CLZ: - return emit_op(compiler, op, (op & SLJIT_I32_OP) ? INT_DATA : WORD_DATA, dst, dstw, TMP_REG1, 0, src, srcw); - } - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, sljit_s32 src1, sljit_sw src1w, sljit_s32 src2, sljit_sw src2w) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_op2(compiler, op, dst, dstw, src1, src1w, src2, src2w)); - ADJUST_LOCAL_OFFSET(dst, dstw); - ADJUST_LOCAL_OFFSET(src1, src1w); - ADJUST_LOCAL_OFFSET(src2, src2w); - - switch (GET_OPCODE(op)) { - case SLJIT_ADD: - case SLJIT_ADDC: - return emit_op(compiler, op, CUMULATIVE_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w); - - case SLJIT_SUB: - case SLJIT_SUBC: - return emit_op(compiler, op, IMM_OP, dst, dstw, src1, src1w, src2, src2w); - - case SLJIT_MUL: - return emit_op(compiler, op, CUMULATIVE_OP, dst, dstw, src1, src1w, src2, src2w); - - case SLJIT_AND: - case SLJIT_OR: - case SLJIT_XOR: - return emit_op(compiler, op, CUMULATIVE_OP | LOGICAL_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w); - - case SLJIT_SHL: - case SLJIT_LSHR: - case SLJIT_ASHR: - if (src2 & SLJIT_IMM) - src2w &= 0x3f; - if (op & SLJIT_I32_OP) - src2w &= 0x1f; - - return emit_op(compiler, op, IMM_OP, dst, dstw, src1, src1w, src2, src2w); - } - - return SLJIT_SUCCESS; -} - -SLJIT_API_FUNC_ATTRIBUTE struct sljit_label * sljit_emit_label(struct sljit_compiler *compiler) -{ - struct sljit_label *label; - - flush_buffer(compiler); - - CHECK_ERROR_PTR(); - CHECK_PTR(check_sljit_emit_label(compiler)); - - if (compiler->last_label && compiler->last_label->size == compiler->size) - return compiler->last_label; - - label = (struct sljit_label *)ensure_abuf(compiler, sizeof(struct sljit_label)); - PTR_FAIL_IF(!label); - set_label(label, compiler); - return label; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw) -{ - sljit_s32 src_r = TMP_REG2; - struct sljit_jump *jump = NULL; - - flush_buffer(compiler); - - CHECK_ERROR(); - CHECK(check_sljit_emit_ijump(compiler, type, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - - if (FAST_IS_REG(src)) { - if (reg_map[src] != 0) - src_r = src; - else - FAIL_IF(ADD_SOLO(TMP_REG2_mapped, reg_map[src], ZERO)); - } - - if (type >= SLJIT_CALL0) { - SLJIT_ASSERT(reg_map[PIC_ADDR_REG] == 16 && PIC_ADDR_REG == TMP_REG2); - if (src & (SLJIT_IMM | SLJIT_MEM)) { - if (src & SLJIT_IMM) - FAIL_IF(emit_const(compiler, reg_map[PIC_ADDR_REG], srcw, 1)); - else { - SLJIT_ASSERT(src_r == TMP_REG2 && (src & SLJIT_MEM)); - FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, TMP_REG2, 0, TMP_REG1, 0, src, srcw)); - } - - FAIL_IF(ADD_SOLO(0, reg_map[SLJIT_R0], ZERO)); - - FAIL_IF(ADDI_SOLO(54, 54, -16)); - - FAIL_IF(JALR_SOLO(reg_map[PIC_ADDR_REG])); - - return ADDI_SOLO(54, 54, 16); - } - - /* Register input. */ - if (type >= SLJIT_CALL1) - FAIL_IF(ADD_SOLO(0, reg_map[SLJIT_R0], ZERO)); - - FAIL_IF(ADD_SOLO(reg_map[PIC_ADDR_REG], reg_map[src_r], ZERO)); - - FAIL_IF(ADDI_SOLO(54, 54, -16)); - - FAIL_IF(JALR_SOLO(reg_map[src_r])); - - return ADDI_SOLO(54, 54, 16); - } - - if (src & SLJIT_IMM) { - jump = (struct sljit_jump *)ensure_abuf(compiler, sizeof(struct sljit_jump)); - FAIL_IF(!jump); - set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_JAL : 0)); - jump->u.target = srcw; - FAIL_IF(emit_const(compiler, TMP_REG2_mapped, 0, 1)); - - if (type >= SLJIT_FAST_CALL) { - FAIL_IF(ADD_SOLO(ZERO, ZERO, ZERO)); - jump->addr = compiler->size; - FAIL_IF(JR_SOLO(reg_map[src_r])); - } else { - jump->addr = compiler->size; - FAIL_IF(JR_SOLO(reg_map[src_r])); - } - - return SLJIT_SUCCESS; - - } else if (src & SLJIT_MEM) { - FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, TMP_REG2, 0, TMP_REG1, 0, src, srcw)); - flush_buffer(compiler); - } - - FAIL_IF(JR_SOLO(reg_map[src_r])); - - if (jump) - jump->addr = compiler->size; - - return SLJIT_SUCCESS; -} - -#define BR_Z(src) \ - inst = BEQZ_X1 | SRCA_X1(src); \ - flags = IS_COND; - -#define BR_NZ(src) \ - inst = BNEZ_X1 | SRCA_X1(src); \ - flags = IS_COND; - -SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump * sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type) -{ - struct sljit_jump *jump; - sljit_ins inst; - sljit_s32 flags = 0; - - flush_buffer(compiler); - - CHECK_ERROR_PTR(); - CHECK_PTR(check_sljit_emit_jump(compiler, type)); - - jump = (struct sljit_jump *)ensure_abuf(compiler, sizeof(struct sljit_jump)); - PTR_FAIL_IF(!jump); - set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP); - type &= 0xff; - - switch (type) { - case SLJIT_EQUAL: - BR_NZ(EQUAL_FLAG); - break; - case SLJIT_NOT_EQUAL: - BR_Z(EQUAL_FLAG); - break; - case SLJIT_LESS: - BR_Z(ULESS_FLAG); - break; - case SLJIT_GREATER_EQUAL: - BR_NZ(ULESS_FLAG); - break; - case SLJIT_GREATER: - BR_Z(UGREATER_FLAG); - break; - case SLJIT_LESS_EQUAL: - BR_NZ(UGREATER_FLAG); - break; - case SLJIT_SIG_LESS: - BR_Z(LESS_FLAG); - break; - case SLJIT_SIG_GREATER_EQUAL: - BR_NZ(LESS_FLAG); - break; - case SLJIT_SIG_GREATER: - BR_Z(GREATER_FLAG); - break; - case SLJIT_SIG_LESS_EQUAL: - BR_NZ(GREATER_FLAG); - break; - case SLJIT_OVERFLOW: - case SLJIT_MUL_OVERFLOW: - BR_Z(OVERFLOW_FLAG); - break; - case SLJIT_NOT_OVERFLOW: - case SLJIT_MUL_NOT_OVERFLOW: - BR_NZ(OVERFLOW_FLAG); - break; - default: - /* Not conditional branch. */ - inst = 0; - break; - } - - jump->flags |= flags; - - if (inst) { - inst = inst | ((type <= SLJIT_JUMP) ? BOFF_X1(5) : BOFF_X1(6)); - PTR_FAIL_IF(PI(inst)); - } - - PTR_FAIL_IF(emit_const(compiler, TMP_REG2_mapped, 0, 1)); - if (type <= SLJIT_JUMP) { - jump->addr = compiler->size; - PTR_FAIL_IF(JR_SOLO(TMP_REG2_mapped)); - } else { - SLJIT_ASSERT(reg_map[PIC_ADDR_REG] == 16 && PIC_ADDR_REG == TMP_REG2); - /* Cannot be optimized out if type is >= CALL0. */ - jump->flags |= IS_JAL | (type >= SLJIT_CALL0 ? SLJIT_REWRITABLE_JUMP : 0); - PTR_FAIL_IF(ADD_SOLO(0, reg_map[SLJIT_R0], ZERO)); - jump->addr = compiler->size; - PTR_FAIL_IF(JALR_SOLO(TMP_REG2_mapped)); - } - - return jump; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, sljit_s32 src, sljit_sw srcw) -{ - SLJIT_UNREACHABLE(); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw dstw, sljit_s32 src1, sljit_sw src1w, sljit_s32 src2, sljit_sw src2w) -{ - SLJIT_UNREACHABLE(); -} - -SLJIT_API_FUNC_ATTRIBUTE struct sljit_const * sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value) -{ - struct sljit_const *const_; - sljit_s32 reg; - - flush_buffer(compiler); - - CHECK_ERROR_PTR(); - CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value)); - ADJUST_LOCAL_OFFSET(dst, dstw); - - const_ = (struct sljit_const *)ensure_abuf(compiler, sizeof(struct sljit_const)); - PTR_FAIL_IF(!const_); - set_const(const_, compiler); - - reg = FAST_IS_REG(dst) ? dst : TMP_REG2; - - PTR_FAIL_IF(emit_const_64(compiler, reg, init_value, 1)); - - if (dst & SLJIT_MEM) - PTR_FAIL_IF(emit_op(compiler, SLJIT_MOV, WORD_DATA, dst, dstw, TMP_REG1, 0, TMP_REG2, 0)); - return const_; -} - -SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target) -{ - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & ~(0xFFFFL << 43)) | (((new_target >> 32) & 0xffff) << 43); - inst[1] = (inst[1] & ~(0xFFFFL << 43)) | (((new_target >> 16) & 0xffff) << 43); - inst[2] = (inst[2] & ~(0xFFFFL << 43)) | ((new_target & 0xffff) << 43); - SLJIT_CACHE_FLUSH(inst, inst + 3); -} - -SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant) -{ - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & ~(0xFFFFL << 43)) | (((new_constant >> 48) & 0xFFFFL) << 43); - inst[1] = (inst[1] & ~(0xFFFFL << 43)) | (((new_constant >> 32) & 0xFFFFL) << 43); - inst[2] = (inst[2] & ~(0xFFFFL << 43)) | (((new_constant >> 16) & 0xFFFFL) << 43); - inst[3] = (inst[3] & ~(0xFFFFL << 43)) | ((new_constant & 0xFFFFL) << 43); - SLJIT_CACHE_FLUSH(inst, inst + 4); -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) -{ - CHECK_REG_INDEX(check_sljit_get_register_index(reg)); - return reg_map[reg]; -} - -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler, - void *instruction, sljit_s32 size) -{ - CHECK_ERROR(); - CHECK(check_sljit_emit_op_custom(compiler, instruction, size)); - return SLJIT_ERR_UNSUPPORTED; -} - diff --git a/dist2/src/sljit/sljitNativeX86_32.c b/dist2/src/sljit/sljitNativeX86_32.c index 074e64b9..79a7e8bb 100644 --- a/dist2/src/sljit/sljitNativeX86_32.c +++ b/dist2/src/sljit/sljitNativeX86_32.c @@ -38,8 +38,10 @@ static sljit_s32 emit_do_imm(struct sljit_compiler *compiler, sljit_u8 opcode, s return SLJIT_SUCCESS; } -static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_s32 type, sljit_sw executable_offset) +static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset) { + sljit_s32 type = jump->flags >> TYPE_SHIFT; + if (type == SLJIT_JUMP) { *code_ptr++ = JMP_i32; jump->addr++; @@ -74,6 +76,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compi CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size)); set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size); + /* Emit ENDBR32 at function entry if needed. */ + FAIL_IF(emit_endbranch(compiler)); + args = get_arg_count(arg_types); compiler->args = args; @@ -305,14 +310,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *comp SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, compiler->local_size)); #endif - size = 2 + (compiler->scratches > 7 ? (compiler->scratches - 7) : 0) + + size = 2 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0) + (compiler->saveds <= 3 ? compiler->saveds : 3); #if (defined SLJIT_X86_32_FASTCALL && SLJIT_X86_32_FASTCALL) if (compiler->args > 2) size += 2; -#else - if (compiler->args > 0) - size += 2; #endif inst = (sljit_u8*)ensure_buf(compiler, 1 + size); FAIL_IF(!inst); @@ -365,6 +367,8 @@ static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_s32 SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3) && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66) && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66)); + /* We don't support (%ebp). */ + SLJIT_ASSERT(!(b & SLJIT_MEM) || immb || reg_map[b & REG_MASK] != 5); size &= 0xf; inst_size = size; @@ -861,14 +865,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return SLJIT_SUCCESS; } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) +static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) { sljit_u8 *inst; - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - CHECK_EXTRA_REGS(src, srcw, (void)0); if (FAST_IS_REG(src)) { @@ -892,3 +892,37 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler RET(); return SLJIT_SUCCESS; } + +static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler) +{ + sljit_s32 size, saved_size; + sljit_s32 has_f64_aligment; + + /* Don't adjust shadow stack if it isn't enabled. */ + if (!cpu_has_shadow_stack ()) + return SLJIT_SUCCESS; + + SLJIT_ASSERT(compiler->args >= 0); + SLJIT_ASSERT(compiler->local_size > 0); + +#if !defined(__APPLE__) + has_f64_aligment = compiler->options & SLJIT_F64_ALIGNMENT; +#else + has_f64_aligment = 0; +#endif + + size = compiler->local_size; + saved_size = (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0) + (compiler->saveds <= 3 ? compiler->saveds : 3)) * sizeof(sljit_uw); + if (has_f64_aligment) { + /* mov TMP_REG1, [esp + local_size]. */ + EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), size); + /* mov TMP_REG1, [TMP_REG1+ saved_size]. */ + EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), saved_size); + /* Move return address to [esp]. */ + EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, TMP_REG1, 0); + size = 0; + } else + size += saved_size; + + return adjust_shadow_stack(compiler, SLJIT_UNUSED, 0, SLJIT_SP, size); +} diff --git a/dist2/src/sljit/sljitNativeX86_64.c b/dist2/src/sljit/sljitNativeX86_64.c index 85065656..e85b56a6 100644 --- a/dist2/src/sljit/sljitNativeX86_64.c +++ b/dist2/src/sljit/sljitNativeX86_64.c @@ -39,8 +39,10 @@ static sljit_s32 emit_load_imm64(struct sljit_compiler *compiler, sljit_s32 reg, return SLJIT_SUCCESS; } -static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_s32 type) +static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr) { + sljit_s32 type = jump->flags >> TYPE_SHIFT; + int short_addr = !(jump->flags & SLJIT_REWRITABLE_JUMP) && !(jump->flags & JUMP_LABEL) && (jump->u.target <= 0xffffffff); /* The relative jump below specialized for this case. */ @@ -72,6 +74,56 @@ static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ return code_ptr; } +static sljit_u8* generate_put_label_code(struct sljit_put_label *put_label, sljit_u8 *code_ptr, sljit_uw max_label) +{ + if (max_label > HALFWORD_MAX) { + put_label->addr -= put_label->flags; + put_label->flags = PATCH_MD; + return code_ptr; + } + + if (put_label->flags == 0) { + /* Destination is register. */ + code_ptr = (sljit_u8*)put_label->addr - 2 - sizeof(sljit_uw); + + SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W); + SLJIT_ASSERT((code_ptr[1] & 0xf8) == MOV_r_i32); + + if ((code_ptr[0] & 0x07) != 0) { + code_ptr[0] = (sljit_u8)(code_ptr[0] & ~0x08); + code_ptr += 2 + sizeof(sljit_s32); + } + else { + code_ptr[0] = code_ptr[1]; + code_ptr += 1 + sizeof(sljit_s32); + } + + put_label->addr = (sljit_uw)code_ptr; + return code_ptr; + } + + code_ptr -= put_label->flags + (2 + sizeof(sljit_uw)); + SLJIT_MEMMOVE(code_ptr, code_ptr + (2 + sizeof(sljit_uw)), put_label->flags); + + SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W); + + if ((code_ptr[1] & 0xf8) == MOV_r_i32) { + code_ptr += 2 + sizeof(sljit_uw); + SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W); + } + + SLJIT_ASSERT(code_ptr[1] == MOV_rm_r); + + code_ptr[0] = (sljit_u8)(code_ptr[0] & ~0x4); + code_ptr[1] = MOV_rm_i32; + code_ptr[2] = (sljit_u8)(code_ptr[2] & ~(0x7 << 3)); + + code_ptr = (sljit_u8*)(put_label->addr - (2 + sizeof(sljit_uw)) + sizeof(sljit_s32)); + put_label->addr = (sljit_uw)code_ptr; + put_label->flags = 0; + return code_ptr; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler, sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size) @@ -83,6 +135,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compi CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size)); set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size); + /* Emit ENDBR64 at function entry if needed. */ + FAIL_IF(emit_endbranch(compiler)); + compiler->mode32 = 0; #ifdef _WIN64 @@ -744,14 +799,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * return SLJIT_SUCCESS; } -SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) +static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw) { sljit_u8 *inst; - CHECK_ERROR(); - CHECK(check_sljit_emit_fast_return(compiler, src, srcw)); - ADJUST_LOCAL_OFFSET(src, srcw); - if (FAST_IS_REG(src)) { if (reg_map[src] < 8) { inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1); @@ -846,3 +897,22 @@ static sljit_s32 emit_mov_int(struct sljit_compiler *compiler, sljit_s32 sign, return SLJIT_SUCCESS; } + +static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler) +{ + sljit_s32 tmp, size; + + /* Don't adjust shadow stack if it isn't enabled. */ + if (!cpu_has_shadow_stack ()) + return SLJIT_SUCCESS; + + size = compiler->local_size; + tmp = compiler->scratches; + if (tmp >= SLJIT_FIRST_SAVED_REG) + size += (tmp - SLJIT_FIRST_SAVED_REG + 1) * sizeof(sljit_uw); + tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG; + if (SLJIT_S0 >= tmp) + size += (SLJIT_S0 - tmp + 1) * sizeof(sljit_uw); + + return adjust_shadow_stack(compiler, SLJIT_UNUSED, 0, SLJIT_SP, size); +} diff --git a/dist2/src/sljit/sljitNativeX86_common.c b/dist2/src/sljit/sljitNativeX86_common.c index 6f02ee3e..ddcc5ebf 100644 --- a/dist2/src/sljit/sljitNativeX86_common.c +++ b/dist2/src/sljit/sljitNativeX86_common.c @@ -428,13 +428,15 @@ static sljit_u8 get_jump_code(sljit_s32 type) } #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_s32 type, sljit_sw executable_offset); +static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset); #else -static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_s32 type); +static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr); +static sljit_u8* generate_put_label_code(struct sljit_put_label *put_label, sljit_u8 *code_ptr, sljit_uw max_label); #endif -static sljit_u8* generate_near_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_s32 type, sljit_sw executable_offset) +static sljit_u8* generate_near_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_u8 *code, sljit_sw executable_offset) { + sljit_s32 type = jump->flags >> TYPE_SHIFT; sljit_s32 short_jump; sljit_uw label_addr; @@ -447,7 +449,7 @@ static sljit_u8* generate_near_jump_code(struct sljit_jump *jump, sljit_u8 *code #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) if ((sljit_sw)(label_addr - (jump->addr + 1)) > HALFWORD_MAX || (sljit_sw)(label_addr - (jump->addr + 1)) < HALFWORD_MIN) - return generate_far_jump_code(jump, code_ptr, type); + return generate_far_jump_code(jump, code_ptr); #endif if (type == SLJIT_JUMP) { @@ -497,13 +499,14 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil struct sljit_label *label; struct sljit_jump *jump; struct sljit_const *const_; + struct sljit_put_label *put_label; CHECK_ERROR_PTR(); CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); /* Second code generation pass. */ - code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size); + code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size, compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -511,6 +514,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil label = compiler->labels; jump = compiler->jumps; const_ = compiler->consts; + put_label = compiler->put_labels; executable_offset = SLJIT_EXEC_OFFSET(code); do { @@ -525,27 +529,38 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil buf_ptr += len; } else { - if (*buf_ptr >= 2) { + switch (*buf_ptr) { + case 0: + label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); + label->size = code_ptr - code; + label = label->next; + break; + case 1: jump->addr = (sljit_uw)code_ptr; if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) - code_ptr = generate_near_jump_code(jump, code_ptr, code, *buf_ptr - 2, executable_offset); + code_ptr = generate_near_jump_code(jump, code_ptr, code, executable_offset); else { #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) - code_ptr = generate_far_jump_code(jump, code_ptr, *buf_ptr - 2, executable_offset); + code_ptr = generate_far_jump_code(jump, code_ptr, executable_offset); #else - code_ptr = generate_far_jump_code(jump, code_ptr, *buf_ptr - 2); + code_ptr = generate_far_jump_code(jump, code_ptr); #endif } jump = jump->next; - } - else if (*buf_ptr == 0) { - label->addr = ((sljit_uw)code_ptr) + executable_offset; - label->size = code_ptr - code; - label = label->next; - } - else { /* *buf_ptr is 1 */ + break; + case 2: const_->addr = ((sljit_uw)code_ptr) - sizeof(sljit_sw); const_ = const_->next; + break; + default: + SLJIT_ASSERT(*buf_ptr == 3); + SLJIT_ASSERT(put_label->label); + put_label->addr = (sljit_uw)code_ptr; +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + code_ptr = generate_put_label_code(put_label, code_ptr, (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size); +#endif + put_label = put_label->next; + break; } buf_ptr++; } @@ -557,6 +572,8 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(!label); SLJIT_ASSERT(!jump); SLJIT_ASSERT(!const_); + SLJIT_ASSERT(!put_label); + SLJIT_ASSERT(code_ptr <= code + compiler->size); jump = compiler->jumps; while (jump) { @@ -591,12 +608,32 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil jump = jump->next; } - /* Some space may be wasted because of short jumps. */ - SLJIT_ASSERT(code_ptr <= code + compiler->size); + put_label = compiler->put_labels; + while (put_label) { +#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) + sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr); +#else + if (put_label->flags & PATCH_MD) { + SLJIT_ASSERT(put_label->label->addr > HALFWORD_MAX); + sljit_unaligned_store_sw((void*)(put_label->addr - sizeof(sljit_sw)), (sljit_sw)put_label->label->addr); + } + else { + SLJIT_ASSERT(put_label->label->addr <= HALFWORD_MAX); + sljit_unaligned_store_s32((void*)(put_label->addr - sizeof(sljit_s32)), (sljit_s32)put_label->label->addr); + } +#endif + + put_label = put_label->next; + } + compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = code_ptr - code; - return (void*)(code + executable_offset); + + code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset); + + SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1); + return (void*)code; } SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) @@ -624,6 +661,9 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) get_cpu_features(); return cpu_has_cmov; + case SLJIT_HAS_PREFETCH: + return 1; + case SLJIT_HAS_SSE2: #if (defined SLJIT_DETECT_SSE2 && SLJIT_DETECT_SSE2) if (cpu_has_sse2 == -1) @@ -669,6 +709,166 @@ static SLJIT_INLINE sljit_s32 emit_sse2_store(struct sljit_compiler *compiler, static SLJIT_INLINE sljit_s32 emit_sse2_load(struct sljit_compiler *compiler, sljit_s32 single, sljit_s32 dst, sljit_s32 src, sljit_sw srcw); +static sljit_s32 emit_cmp_binary(struct sljit_compiler *compiler, + sljit_s32 src1, sljit_sw src1w, + sljit_s32 src2, sljit_sw src2w); + +static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler) +{ +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) + /* Emit endbr32/endbr64 when CET is enabled. */ + sljit_u8 *inst; + inst = (sljit_u8*)ensure_buf(compiler, 1 + 4); + FAIL_IF(!inst); + INC_SIZE(4); + *inst++ = 0xf3; + *inst++ = 0x0f; + *inst++ = 0x1e; +#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) + *inst = 0xfb; +#else + *inst = 0xfa; +#endif +#else /* !SLJIT_CONFIG_X86_CET */ + SLJIT_UNUSED_ARG(compiler); +#endif /* SLJIT_CONFIG_X86_CET */ + return SLJIT_SUCCESS; +} + +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) + +static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg) +{ + sljit_u8 *inst; + sljit_s32 size; + +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + size = 5; +#else + size = 4; +#endif + + inst = (sljit_u8*)ensure_buf(compiler, 1 + size); + FAIL_IF(!inst); + INC_SIZE(size); + *inst++ = 0xf3; +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B); +#endif + *inst++ = 0x0f; + *inst++ = 0x1e; + *inst = (0x3 << 6) | (0x1 << 3) | (reg_map[reg] & 0x7); + return SLJIT_SUCCESS; +} + +static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg) +{ + sljit_u8 *inst; + sljit_s32 size; + +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + size = 5; +#else + size = 4; +#endif + + inst = (sljit_u8*)ensure_buf(compiler, 1 + size); + FAIL_IF(!inst); + INC_SIZE(size); + *inst++ = 0xf3; +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + *inst++ = REX_W | (reg_map[reg] <= 7 ? 0 : REX_B); +#endif + *inst++ = 0x0f; + *inst++ = 0xae; + *inst = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7); + return SLJIT_SUCCESS; +} + +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ + +static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void) +{ +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) + return _get_ssp() != 0; +#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */ + return 0; +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ +} + +static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler, + sljit_s32 src, sljit_sw srcw, sljit_s32 base, sljit_sw disp) +{ +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) + sljit_u8 *inst, *jz_after_cmp_inst; + sljit_uw size_jz_after_cmp_inst; + + sljit_uw size_before_rdssp_inst = compiler->size; + + /* Generate "RDSSP TMP_REG1". */ + FAIL_IF(emit_rdssp(compiler, TMP_REG1)); + + /* Load return address on shadow stack into TMP_REG1. */ +#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) + SLJIT_ASSERT(reg_map[TMP_REG1] == 5); + + /* Hand code unsupported "mov 0x0(%ebp),%ebp". */ + inst = (sljit_u8*)ensure_buf(compiler, 1 + 3); + FAIL_IF(!inst); + INC_SIZE(3); + *inst++ = 0x8b; + *inst++ = 0x6d; + *inst = 0; +#else /* !SLJIT_CONFIG_X86_32 */ + EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(TMP_REG1), 0); +#endif /* SLJIT_CONFIG_X86_32 */ + + if (src == SLJIT_UNUSED) { + /* Return address is on stack. */ + src = SLJIT_MEM1(base); + srcw = disp; + } + + /* Compare return address against TMP_REG1. */ + FAIL_IF(emit_cmp_binary (compiler, TMP_REG1, 0, src, srcw)); + + /* Generate JZ to skip shadow stack ajdustment when shadow + stack matches normal stack. */ + inst = (sljit_u8*)ensure_buf(compiler, 1 + 2); + FAIL_IF(!inst); + INC_SIZE(2); + *inst++ = get_jump_code(SLJIT_EQUAL) - 0x10; + size_jz_after_cmp_inst = compiler->size; + jz_after_cmp_inst = inst; + +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + /* REX_W is not necessary. */ + compiler->mode32 = 1; +#endif + /* Load 1 into TMP_REG1. */ + EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, 1); + + /* Generate "INCSSP TMP_REG1". */ + FAIL_IF(emit_incssp(compiler, TMP_REG1)); + + /* Jump back to "RDSSP TMP_REG1" to check shadow stack again. */ + inst = (sljit_u8*)ensure_buf(compiler, 1 + 2); + FAIL_IF(!inst); + INC_SIZE(2); + *inst++ = JMP_i8; + *inst = size_before_rdssp_inst - compiler->size; + + *jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst; +#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */ + SLJIT_UNUSED_ARG(compiler); + SLJIT_UNUSED_ARG(src); + SLJIT_UNUSED_ARG(srcw); + SLJIT_UNUSED_ARG(base); + SLJIT_UNUSED_ARG(disp); +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ + return SLJIT_SUCCESS; +} + #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) #include "sljitNativeX86_32.c" #else @@ -872,6 +1072,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile EMIT_MOV(compiler, SLJIT_R1, 0, TMP_REG1, 0); #endif break; + case SLJIT_ENDBR: + return emit_endbranch(compiler); + case SLJIT_SKIP_FRAMES_BEFORE_RETURN: + return skip_frames_before_return(compiler); } return SLJIT_SUCCESS; @@ -1041,12 +1245,12 @@ static sljit_s32 emit_prefetch(struct sljit_compiler *compiler, sljit_s32 op, *inst++ = GROUP_0F; *inst++ = PREFETCH; - if (op >= SLJIT_MOV_U8 && op <= SLJIT_MOV_S8) - *inst |= (3 << 3); - else if (op >= SLJIT_MOV_U16 && op <= SLJIT_MOV_S16) - *inst |= (2 << 3); - else + if (op == SLJIT_PREFETCH_L1) *inst |= (1 << 3); + else if (op == SLJIT_PREFETCH_L2) + *inst |= (2 << 3); + else if (op == SLJIT_PREFETCH_L3) + *inst |= (3 << 3); return SLJIT_SUCCESS; } @@ -1251,12 +1455,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compile compiler->mode32 = op_flags & SLJIT_I32_OP; #endif - if (dst == SLJIT_UNUSED && !HAS_FLAGS(op)) { - if (op <= SLJIT_MOV_P && (src & SLJIT_MEM)) - return emit_prefetch(compiler, op, src, srcw); - return SLJIT_SUCCESS; - } - op = GET_OPCODE(op); if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) { @@ -2117,6 +2315,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile if (!HAS_FLAGS(op)) { if ((src2 & SLJIT_IMM) && emit_lea_binary(compiler, dst, dstw, src1, src1w, SLJIT_IMM, -src2w) != SLJIT_ERR_UNSUPPORTED) return compiler->error; + if (SLOW_IS_REG(dst) && src2 == dst) { + FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB), dst, 0, dst, 0, src1, src1w)); + return emit_unary(compiler, NEG_rm, dst, 0, dst, 0); + } } if (dst == SLJIT_UNUSED) @@ -2153,6 +2355,33 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compile return SLJIT_SUCCESS; } +SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op, + sljit_s32 src, sljit_sw srcw) +{ + CHECK_ERROR(); + CHECK(check_sljit_emit_op_src(compiler, op, src, srcw)); + ADJUST_LOCAL_OFFSET(src, srcw); + + CHECK_EXTRA_REGS(src, srcw, (void)0); + + switch (op) { + case SLJIT_FAST_RETURN: + return emit_fast_return(compiler, src, srcw); + case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN: + /* Don't adjust shadow stack if it isn't enabled. */ + if (!cpu_has_shadow_stack ()) + return SLJIT_SUCCESS; + return adjust_shadow_stack(compiler, src, srcw, SLJIT_UNUSED, 0); + case SLJIT_PREFETCH_L1: + case SLJIT_PREFETCH_L2: + case SLJIT_PREFETCH_L3: + case SLJIT_PREFETCH_ONCE: + return emit_prefetch(compiler, op, src, srcw); + } + + return SLJIT_SUCCESS; +} + SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg) { CHECK_REG_INDEX(check_sljit_get_register_index(reg)); @@ -2481,7 +2710,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compile jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump)); PTR_FAIL_IF_NULL(jump); - set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP); + set_jump(jump, compiler, (type & SLJIT_REWRITABLE_JUMP) | ((type & 0xff) << TYPE_SHIFT)); type &= 0xff; /* Worst case size. */ @@ -2495,7 +2724,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compile PTR_FAIL_IF_NULL(inst); *inst++ = 0; - *inst++ = type + 2; + *inst++ = 1; return jump; } @@ -2513,7 +2742,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compi if (src == SLJIT_IMM) { jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump)); FAIL_IF_NULL(jump); - set_jump(jump, compiler, JUMP_ADDR); + set_jump(jump, compiler, JUMP_ADDR | (type << TYPE_SHIFT)); jump->u.target = srcw; /* Worst case size. */ @@ -2527,7 +2756,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compi FAIL_IF_NULL(inst); *inst++ = 0; - *inst++ = type + 2; + *inst++ = 1; } else { #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) @@ -2831,7 +3060,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi PTR_FAIL_IF(!inst); *inst++ = 0; - *inst++ = 1; + *inst++ = 2; #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) if (dst & SLJIT_MEM) @@ -2842,18 +3071,72 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compi return const_; } +SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw) +{ + struct sljit_put_label *put_label; + sljit_u8 *inst; +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + sljit_s32 reg; + sljit_uw start_size; +#endif + + CHECK_ERROR_PTR(); + CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw)); + ADJUST_LOCAL_OFFSET(dst, dstw); + + CHECK_EXTRA_REGS(dst, dstw, (void)0); + + put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label)); + PTR_FAIL_IF(!put_label); + set_put_label(put_label, compiler, 0); + +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + compiler->mode32 = 0; + reg = FAST_IS_REG(dst) ? dst : TMP_REG1; + + if (emit_load_imm64(compiler, reg, 0)) + return NULL; +#else + if (emit_mov(compiler, dst, dstw, SLJIT_IMM, 0)) + return NULL; +#endif + +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + if (dst & SLJIT_MEM) { + start_size = compiler->size; + if (emit_mov(compiler, dst, dstw, TMP_REG1, 0)) + return NULL; + put_label->flags = compiler->size - start_size; + } +#endif + + inst = (sljit_u8*)ensure_buf(compiler, 2); + PTR_FAIL_IF(!inst); + + *inst++ = 0; + *inst++ = 3; + + return put_label; +} + SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0); #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) sljit_unaligned_store_sw((void*)addr, new_target - (addr + 4) - (sljit_uw)executable_offset); #else sljit_unaligned_store_sw((void*)addr, (sljit_sw) new_target); #endif + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 0); sljit_unaligned_store_sw((void*)addr, new_constant); + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 1); } diff --git a/dist2/src/sljit/sljitProtExecAllocator.c b/dist2/src/sljit/sljitProtExecAllocator.c index 8a5b2b3c..147175af 100644 --- a/dist2/src/sljit/sljitProtExecAllocator.c +++ b/dist2/src/sljit/sljitProtExecAllocator.c @@ -70,92 +70,112 @@ struct chunk_header { void *executable; - int fd; }; /* alloc_chunk / free_chunk : * allocate executable system memory chunks * the size is always divisible by CHUNK_SIZE - allocator_grab_lock / allocator_release_lock : - * make the allocator thread safe - * can be empty if the OS (or the application) does not support threading + SLJIT_ALLOCATOR_LOCK / SLJIT_ALLOCATOR_UNLOCK : + * provided as part of sljitUtils * only the allocator requires this lock, sljit is fully thread safe as it only uses local variables */ +#ifndef __NetBSD__ +#include <sys/stat.h> #include <fcntl.h> +#include <stdio.h> +#include <string.h> #ifndef O_NOATIME #define O_NOATIME 0 #endif -#ifdef __O_TMPFILE +/* this is a linux extension available since kernel 3.11 */ #ifndef O_TMPFILE -#define O_TMPFILE (__O_TMPFILE | O_DIRECTORY) -#endif +#define O_TMPFILE 020200000 #endif -int mkostemp(char *template, int flags); +#ifndef _GNU_SOURCE char *secure_getenv(const char *name); +int mkostemp(char *template, int flags); +#endif static SLJIT_INLINE int create_tempfile(void) { int fd; - char tmp_name[256]; - size_t tmp_name_len; + size_t tmp_name_len = 0; char *dir; - size_t len; - -#ifdef P_tmpdir - len = (P_tmpdir != NULL) ? strlen(P_tmpdir) : 0; + struct stat st; +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + mode_t mode; +#endif - if (len > 0 && len < sizeof(tmp_name)) { - strcpy(tmp_name, P_tmpdir); - tmp_name_len = len; - } - else { - strcpy(tmp_name, "/tmp"); - tmp_name_len = 4; +#ifdef HAVE_MEMFD_CREATE + /* this is a GNU extension, make sure to use -D_GNU_SOURCE */ + fd = memfd_create("sljit", MFD_CLOEXEC); + if (fd != -1) { + fchmod(fd, 0); + return fd; } -#else - strcpy(tmp_name, "/tmp"); - tmp_name_len = 4; #endif dir = secure_getenv("TMPDIR"); + if (dir) { - len = strlen(dir); - if (len > 0 && len < sizeof(tmp_name)) { - strcpy(tmp_name, dir); - tmp_name_len = len; + tmp_name_len = strlen(dir); + if (tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)) { + if ((stat(dir, &st) == 0) && S_ISDIR(st.st_mode)) + strcpy(tmp_name, dir); } } +#ifdef P_tmpdir + if (!tmp_name_len) { + tmp_name_len = strlen(P_tmpdir); + if (tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)) + strcpy(tmp_name, P_tmpdir); + } +#endif + if (!tmp_name_len) { + strcpy(tmp_name, "/tmp"); + tmp_name_len = 4; + } + SLJIT_ASSERT(tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)); - while (tmp_name_len > 0 && tmp_name[tmp_name_len - 1] == '/') { - tmp_name_len--; - tmp_name[tmp_name_len] = '\0'; - } + if (tmp_name[tmp_name_len - 1] == '/') + tmp_name[--tmp_name_len] = '\0'; -#ifdef O_TMPFILE - fd = open(tmp_name, O_TMPFILE | O_EXCL | O_RDWR | O_NOATIME | O_CLOEXEC, S_IRUSR | S_IWUSR); +#ifdef __linux__ + /* + * the previous trimming might had left an empty string if TMPDIR="/" + * so work around the problem below + */ + fd = open(tmp_name_len ? tmp_name : "/", + O_TMPFILE | O_EXCL | O_RDWR | O_NOATIME | O_CLOEXEC, 0); if (fd != -1) return fd; #endif if (tmp_name_len + 7 >= sizeof(tmp_name)) - { return -1; - } strcpy(tmp_name + tmp_name_len, "/XXXXXX"); +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + mode = umask(0777); +#endif fd = mkostemp(tmp_name, O_CLOEXEC | O_NOATIME); +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + umask(mode); +#else + fchmod(fd, 0); +#endif if (fd == -1) - return fd; + return -1; if (unlink(tmp_name)) { close(fd); @@ -189,23 +209,52 @@ static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) retval->executable = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); if (retval->executable == MAP_FAILED) { - munmap(retval, size); + munmap((void *)retval, size); close(fd); return NULL; } - retval->fd = fd; + close(fd); return retval; } +#else +/* + * MAP_REMAPDUP is a NetBSD extension available sinde 8.0, make sure to + * adjust your feature macros (ex: -D_NETBSD_SOURCE) as needed + */ +static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) +{ + struct chunk_header *retval; + + retval = (struct chunk_header *)mmap(NULL, size, + PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC), + MAP_ANON | MAP_SHARED, -1, 0); + + if (retval == MAP_FAILED) + return NULL; + + retval->executable = mremap(retval, size, NULL, size, MAP_REMAPDUP); + if (retval->executable == MAP_FAILED) { + munmap((void *)retval, size); + return NULL; + } + + if (mprotect(retval->executable, size, PROT_READ | PROT_EXEC) == -1) { + munmap(retval->executable, size); + munmap((void *)retval, size); + return NULL; + } + + return retval; +} +#endif /* NetBSD */ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) { struct chunk_header *header = ((struct chunk_header *)chunk) - 1; - int fd = header->fd; munmap(header->executable, size); - munmap(header, size); - close(fd); + munmap((void *)header, size); } /* --------------------------------------------------------------------- */ @@ -272,7 +321,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) sljit_uw chunk_size; sljit_sw executable_offset; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); if (size < (64 - sizeof(struct block_header))) size = (64 - sizeof(struct block_header)); size = ALIGN_SIZE(size); @@ -297,7 +346,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } allocated_size += size; header->size = size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } free_block = free_block->next; @@ -308,7 +357,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) chunk_header = alloc_chunk(chunk_size); if (!chunk_header) { - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return NULL; } @@ -342,7 +391,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) next_header->size = 1; next_header->prev_size = chunk_size; next_header->executable_offset = executable_offset; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } @@ -351,7 +400,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) struct block_header *header; struct free_block* free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); header = AS_BLOCK_HEADER(ptr, -(sljit_sw)sizeof(struct block_header)); header = AS_BLOCK_HEADER(header, -header->executable_offset); allocated_size -= header->size; @@ -385,11 +434,13 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) if (total_size - free_block->size > (allocated_size * 3 / 2)) { total_size -= free_block->size; sljit_remove_free_block(free_block); - free_chunk(free_block, free_block->size + sizeof(struct block_header)); + free_chunk(free_block, free_block->size + + sizeof(struct chunk_header) + + sizeof(struct block_header)); } } - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) @@ -397,7 +448,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) struct free_block* free_block; struct free_block* next_free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); free_block = free_blocks; while (free_block) { @@ -406,13 +457,15 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) AS_BLOCK_HEADER(free_block, free_block->size)->size == 1) { total_size -= free_block->size; sljit_remove_free_block(free_block); - free_chunk(free_block, free_block->size + sizeof(struct block_header)); + free_chunk(free_block, free_block->size + + sizeof(struct chunk_header) + + sizeof(struct block_header)); } free_block = next_free_block; } SLJIT_ASSERT((total_size && free_blocks) || (!total_size && !free_blocks)); - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr) diff --git a/dist2/src/sljit/sljitUtils.c b/dist2/src/sljit/sljitUtils.c index 5c2a8389..08ca35cf 100644 --- a/dist2/src/sljit/sljitUtils.c +++ b/dist2/src/sljit/sljitUtils.c @@ -28,214 +28,225 @@ /* Locks */ /* ------------------------------------------------------------------------ */ -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) || (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) +/* Executable Allocator */ +#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) \ + && !(defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR) #if (defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) +#define SLJIT_ALLOCATOR_LOCK() +#define SLJIT_ALLOCATOR_UNLOCK() +#elif !(defined _WIN32) +#include <pthread.h> + +static pthread_mutex_t allocator_lock = PTHREAD_MUTEX_INITIALIZER; -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) +#define SLJIT_ALLOCATOR_LOCK() pthread_mutex_lock(&allocator_lock) +#define SLJIT_ALLOCATOR_UNLOCK() pthread_mutex_unlock(&allocator_lock) +#else /* windows */ +static HANDLE allocator_lock; static SLJIT_INLINE void allocator_grab_lock(void) { - /* Always successful. */ + HANDLE lock; + if (SLJIT_UNLIKELY(!allocator_lock)) { + lock = CreateMutex(NULL, FALSE, NULL); + if (InterlockedCompareExchangePointer(&allocator_lock, lock, NULL)) + CloseHandle(lock); + } + WaitForSingleObject(allocator_lock, INFINITE); } -static SLJIT_INLINE void allocator_release_lock(void) -{ - /* Always successful. */ -} +#define SLJIT_ALLOCATOR_LOCK() allocator_grab_lock() +#define SLJIT_ALLOCATOR_UNLOCK() ReleaseMutex(allocator_lock) +#endif /* thread implementation */ +#endif /* SLJIT_EXECUTABLE_ALLOCATOR && !SLJIT_WX_EXECUTABLE_ALLOCATOR */ -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ +/* ------------------------------------------------------------------------ */ +/* Stack */ +/* ------------------------------------------------------------------------ */ -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) +#if ((defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) \ + && !(defined SLJIT_UTIL_SIMPLE_STACK_ALLOCATION && SLJIT_UTIL_SIMPLE_STACK_ALLOCATION)) \ + || ((defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) \ + && !((defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) \ + || (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR))) -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) -{ - /* Always successful. */ -} +#ifndef _WIN32 +/* Provides mmap function. */ +#include <sys/types.h> +#include <sys/mman.h> -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) -{ - /* Always successful. */ -} +#ifndef MAP_ANON +#ifdef MAP_ANONYMOUS +#define MAP_ANON MAP_ANONYMOUS +#endif /* MAP_ANONYMOUS */ +#endif /* !MAP_ANON */ -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ +#ifndef MAP_ANON -#elif defined(_WIN32) /* SLJIT_SINGLE_THREADED */ +#include <fcntl.h> -#include "windows.h" +#ifdef O_CLOEXEC +#define SLJIT_CLOEXEC O_CLOEXEC +#else /* !O_CLOEXEC */ +#define SLJIT_CLOEXEC 0 +#endif /* O_CLOEXEC */ -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) +/* Some old systems do not have MAP_ANON. */ +static int dev_zero = -1; -static HANDLE allocator_mutex = 0; +#if (defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) -static SLJIT_INLINE void allocator_grab_lock(void) +static SLJIT_INLINE int open_dev_zero(void) { - /* No idea what to do if an error occures. Static mutexes should never fail... */ - if (!allocator_mutex) - allocator_mutex = CreateMutex(NULL, TRUE, NULL); - else - WaitForSingleObject(allocator_mutex, INFINITE); -} + dev_zero = open("/dev/zero", O_RDWR | SLJIT_CLOEXEC); -static SLJIT_INLINE void allocator_release_lock(void) -{ - ReleaseMutex(allocator_mutex); + return dev_zero < 0; } -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ +#else /* !SLJIT_SINGLE_THREADED */ -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) +#include <pthread.h> -static HANDLE global_mutex = 0; +static pthread_mutex_t dev_zero_mutex = PTHREAD_MUTEX_INITIALIZER; -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) +static SLJIT_INLINE int open_dev_zero(void) { - /* No idea what to do if an error occures. Static mutexes should never fail... */ - if (!global_mutex) - global_mutex = CreateMutex(NULL, TRUE, NULL); - else - WaitForSingleObject(global_mutex, INFINITE); -} + pthread_mutex_lock(&dev_zero_mutex); + if (SLJIT_UNLIKELY(dev_zero < 0)) + dev_zero = open("/dev/zero", O_RDWR | SLJIT_CLOEXEC); -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) -{ - ReleaseMutex(global_mutex); + pthread_mutex_unlock(&dev_zero_mutex); + return dev_zero < 0; } -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ - -#else /* _WIN32 */ - -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) - -#include <pthread.h> +#endif /* SLJIT_SINGLE_THREADED */ +#undef SLJIT_CLOEXEC +#endif /* !MAP_ANON */ +#endif /* !_WIN32 */ +#endif /* open_dev_zero */ -static pthread_mutex_t allocator_mutex = PTHREAD_MUTEX_INITIALIZER; +#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) \ + || (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) -static SLJIT_INLINE void allocator_grab_lock(void) -{ - pthread_mutex_lock(&allocator_mutex); -} +#ifdef _WIN32 -static SLJIT_INLINE void allocator_release_lock(void) -{ - pthread_mutex_unlock(&allocator_mutex); +static SLJIT_INLINE sljit_sw get_page_alignment(void) { + SYSTEM_INFO si; + static sljit_sw sljit_page_align; + if (!sljit_page_align) { + GetSystemInfo(&si); + sljit_page_align = si.dwPageSize - 1; + } + return sljit_page_align; } -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ - -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) - -#include <pthread.h> +#else -static pthread_mutex_t global_mutex = PTHREAD_MUTEX_INITIALIZER; +#include <unistd.h> -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) -{ - pthread_mutex_lock(&global_mutex); +static SLJIT_INLINE sljit_sw get_page_alignment(void) { + static sljit_sw sljit_page_align; + if (!sljit_page_align) { + sljit_page_align = sysconf(_SC_PAGESIZE); + /* Should never happen. */ + if (sljit_page_align < 0) + sljit_page_align = 4096; + sljit_page_align--; + } + return sljit_page_align; } -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) -{ - pthread_mutex_unlock(&global_mutex); -} +#endif /* _WIN32 */ -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ +#endif /* get_page_alignment() */ -#endif /* _WIN32 */ +#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) -/* ------------------------------------------------------------------------ */ -/* Stack */ -/* ------------------------------------------------------------------------ */ +#if (defined SLJIT_UTIL_SIMPLE_STACK_ALLOCATION && SLJIT_UTIL_SIMPLE_STACK_ALLOCATION) -#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) || (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) +SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(sljit_uw start_size, sljit_uw max_size, void *allocator_data) +{ + struct sljit_stack *stack; + void *ptr; -#ifdef _WIN32 -#include "windows.h" -#else -/* Provides mmap function. */ -#include <sys/mman.h> -/* For detecting the page size. */ -#include <unistd.h> + SLJIT_UNUSED_ARG(allocator_data); -#ifndef MAP_ANON + if (start_size > max_size || start_size < 1) + return NULL; -#include <fcntl.h> + stack = (struct sljit_stack*)SLJIT_MALLOC(sizeof(struct sljit_stack), allocator_data); + if (stack == NULL) + return NULL; -/* Some old systems does not have MAP_ANON. */ -static sljit_s32 dev_zero = -1; + ptr = SLJIT_MALLOC(max_size, allocator_data); + if (ptr == NULL) { + SLJIT_FREE(stack, allocator_data); + return NULL; + } -#if (defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) + stack->min_start = (sljit_u8 *)ptr; + stack->end = stack->min_start + max_size; + stack->start = stack->end - start_size; + stack->top = stack->end; + return stack; +} -static SLJIT_INLINE sljit_s32 open_dev_zero(void) +SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) { - dev_zero = open("/dev/zero", O_RDWR); - return dev_zero < 0; + SLJIT_UNUSED_ARG(allocator_data); + SLJIT_FREE((void*)stack->min_start, allocator_data); + SLJIT_FREE(stack, allocator_data); } -#else /* SLJIT_SINGLE_THREADED */ - -#include <pthread.h> - -static pthread_mutex_t dev_zero_mutex = PTHREAD_MUTEX_INITIALIZER; - -static SLJIT_INLINE sljit_s32 open_dev_zero(void) +SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_stack *stack, sljit_u8 *new_start) { - pthread_mutex_lock(&dev_zero_mutex); - /* The dev_zero might be initialized by another thread during the waiting. */ - if (dev_zero < 0) { - dev_zero = open("/dev/zero", O_RDWR); - } - pthread_mutex_unlock(&dev_zero_mutex); - return dev_zero < 0; + if ((new_start < stack->min_start) || (new_start >= stack->end)) + return NULL; + stack->start = new_start; + return new_start; } -#endif /* SLJIT_SINGLE_THREADED */ +#else /* !SLJIT_UTIL_SIMPLE_STACK_ALLOCATION */ -#endif +#ifdef _WIN32 -#endif +SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) +{ + SLJIT_UNUSED_ARG(allocator_data); + VirtualFree((void*)stack->min_start, 0, MEM_RELEASE); + SLJIT_FREE(stack, allocator_data); +} -#endif /* SLJIT_UTIL_STACK || SLJIT_EXECUTABLE_ALLOCATOR */ +#else /* !_WIN32 */ -#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) +SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) +{ + SLJIT_UNUSED_ARG(allocator_data); + munmap((void*)stack->min_start, stack->end - stack->min_start); + SLJIT_FREE(stack, allocator_data); +} -/* Planning to make it even more clever in the future. */ -static sljit_sw sljit_page_align = 0; +#endif /* _WIN32 */ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(sljit_uw start_size, sljit_uw max_size, void *allocator_data) { struct sljit_stack *stack; void *ptr; -#ifdef _WIN32 - SYSTEM_INFO si; -#endif + sljit_sw page_align; SLJIT_UNUSED_ARG(allocator_data); + if (start_size > max_size || start_size < 1) return NULL; -#ifdef _WIN32 - if (!sljit_page_align) { - GetSystemInfo(&si); - sljit_page_align = si.dwPageSize - 1; - } -#else - if (!sljit_page_align) { - sljit_page_align = sysconf(_SC_PAGESIZE); - /* Should never happen. */ - if (sljit_page_align < 0) - sljit_page_align = 4096; - sljit_page_align--; - } -#endif - stack = (struct sljit_stack*)SLJIT_MALLOC(sizeof(struct sljit_stack), allocator_data); - if (!stack) + if (stack == NULL) return NULL; /* Align max_size. */ - max_size = (max_size + sljit_page_align) & ~sljit_page_align; + page_align = get_page_alignment(); + max_size = (max_size + page_align) & ~page_align; #ifdef _WIN32 ptr = VirtualAlloc(NULL, max_size, MEM_RESERVE, PAGE_READWRITE); @@ -252,18 +263,16 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(slj sljit_free_stack(stack, allocator_data); return NULL; } -#else +#else /* !_WIN32 */ #ifdef MAP_ANON ptr = mmap(NULL, max_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); -#else - if (dev_zero < 0) { - if (open_dev_zero()) { - SLJIT_FREE(stack, allocator_data); - return NULL; - } +#else /* !MAP_ANON */ + if (SLJIT_UNLIKELY((dev_zero < 0) && open_dev_zero())) { + SLJIT_FREE(stack, allocator_data); + return NULL; } ptr = mmap(NULL, max_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, dev_zero, 0); -#endif +#endif /* MAP_ANON */ if (ptr == MAP_FAILED) { SLJIT_FREE(stack, allocator_data); return NULL; @@ -271,35 +280,28 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(slj stack->min_start = (sljit_u8 *)ptr; stack->end = stack->min_start + max_size; stack->start = stack->end - start_size; -#endif +#endif /* _WIN32 */ + stack->top = stack->end; return stack; } -#undef PAGE_ALIGN - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) -{ - SLJIT_UNUSED_ARG(allocator_data); -#ifdef _WIN32 - VirtualFree((void*)stack->min_start, 0, MEM_RELEASE); -#else - munmap((void*)stack->min_start, stack->end - stack->min_start); -#endif - SLJIT_FREE(stack, allocator_data); -} - SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_stack *stack, sljit_u8 *new_start) { +#if defined _WIN32 || defined(POSIX_MADV_DONTNEED) sljit_uw aligned_old_start; sljit_uw aligned_new_start; + sljit_sw page_align; +#endif if ((new_start < stack->min_start) || (new_start >= stack->end)) return NULL; #ifdef _WIN32 - aligned_new_start = (sljit_uw)new_start & ~sljit_page_align; - aligned_old_start = ((sljit_uw)stack->start) & ~sljit_page_align; + page_align = get_page_alignment(); + + aligned_new_start = (sljit_uw)new_start & ~page_align; + aligned_old_start = ((sljit_uw)stack->start) & ~page_align; if (aligned_new_start != aligned_old_start) { if (aligned_new_start < aligned_old_start) { if (!VirtualAlloc((void*)aligned_new_start, aligned_old_start - aligned_new_start, MEM_COMMIT, PAGE_READWRITE)) @@ -310,24 +312,26 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_st return NULL; } } -#else - if (stack->start < new_start) { - aligned_new_start = (sljit_uw)new_start & ~sljit_page_align; - aligned_old_start = ((sljit_uw)stack->start) & ~sljit_page_align; - /* If madvise is available, we release the unnecessary space. */ -#if defined(MADV_DONTNEED) - if (aligned_new_start > aligned_old_start) - madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, MADV_DONTNEED); #elif defined(POSIX_MADV_DONTNEED) - if (aligned_new_start > aligned_old_start) + if (stack->start < new_start) { + page_align = get_page_alignment(); + + aligned_new_start = (sljit_uw)new_start & ~page_align; + aligned_old_start = ((sljit_uw)stack->start) & ~page_align; + + if (aligned_new_start > aligned_old_start) { posix_madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, POSIX_MADV_DONTNEED); -#endif +#ifdef MADV_FREE + madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, MADV_FREE); +#endif /* MADV_FREE */ + } } -#endif +#endif /* _WIN32 */ + stack->start = new_start; return new_start; } -#endif /* SLJIT_UTIL_STACK */ +#endif /* SLJIT_UTIL_SIMPLE_STACK_ALLOCATION */ -#endif +#endif /* SLJIT_UTIL_STACK */ diff --git a/dist2/src/sljit/sljitWXExecAllocator.c b/dist2/src/sljit/sljitWXExecAllocator.c new file mode 100644 index 00000000..6ef71f7d --- /dev/null +++ b/dist2/src/sljit/sljitWXExecAllocator.c @@ -0,0 +1,225 @@ +/* + * Stack-less Just-In-Time compiler + * + * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + This file contains a simple W^X executable memory allocator for POSIX + like systems and Windows + + In *NIX, MAP_ANON is required (that is considered a feature) so make + sure to set the right availability macros for your system or the code + will fail to build. + + If your system doesn't support mapping of anonymous pages (ex: IRIX) it + is also likely that it doesn't need this allocator and should be using + the standard one instead. + + It allocates a separate map for each code block and may waste a lot of + memory, because whatever was requested, will be rounded up to the page + size (minimum 4KB, but could be even bigger). + + It changes the page permissions (RW <-> RX) as needed and therefore, if you + will be updating the code after it has been generated, need to make sure to + block any concurrent execution, or could result in a SIGBUS, that could + even manifest itself at a different address than the one that was being + modified. + + Only use if you are unable to use the regular allocator because of security + restrictions and adding exceptions to your application or the system are + not possible. +*/ + +#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) \ + sljit_update_wx_flags((from), (to), (enable_exec)) + +#ifndef _WIN32 +#include <sys/types.h> +#include <sys/mman.h> + +#ifdef __NetBSD__ +#if defined(PROT_MPROTECT) +#define check_se_protected(ptr, size) (0) +#define SLJIT_PROT_WX PROT_MPROTECT(PROT_EXEC) +#else /* !PROT_MPROTECT */ +#ifdef _NETBSD_SOURCE +#include <sys/param.h> +#else /* !_NETBSD_SOURCE */ +typedef unsigned int u_int; +#define devmajor_t sljit_s32 +#endif /* _NETBSD_SOURCE */ +#include <sys/sysctl.h> +#include <unistd.h> + +#define check_se_protected(ptr, size) netbsd_se_protected() + +static SLJIT_INLINE int netbsd_se_protected(void) +{ + int mib[3]; + int paxflags; + size_t len = sizeof(paxflags); + + mib[0] = CTL_PROC; + mib[1] = getpid(); + mib[2] = PROC_PID_PAXFLAGS; + + if (SLJIT_UNLIKELY(sysctl(mib, 3, &paxflags, &len, NULL, 0) < 0)) + return -1; + + return (paxflags & CTL_PROC_PAXFLAGS_MPROTECT) ? -1 : 0; +} +#endif /* PROT_MPROTECT */ +#else /* POSIX */ +#define check_se_protected(ptr, size) generic_se_protected(ptr, size) + +static SLJIT_INLINE int generic_se_protected(void *ptr, sljit_uw size) +{ + if (SLJIT_LIKELY(!mprotect(ptr, size, PROT_EXEC))) + return mprotect(ptr, size, PROT_READ | PROT_WRITE); + + return -1; +} +#endif /* NetBSD */ + +#if defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED +#define SLJIT_SE_LOCK() +#define SLJIT_SE_UNLOCK() +#else /* !SLJIT_SINGLE_THREADED */ +#include <pthread.h> +#define SLJIT_SE_LOCK() pthread_mutex_lock(&se_lock) +#define SLJIT_SE_UNLOCK() pthread_mutex_unlock(&se_lock) +#endif /* SLJIT_SINGLE_THREADED */ + +#ifndef SLJIT_PROT_WX +#define SLJIT_PROT_WX 0 +#endif /* !SLJIT_PROT_WX */ + +SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) +{ +#if !(defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) + static pthread_mutex_t se_lock = PTHREAD_MUTEX_INITIALIZER; +#endif + static int se_protected = !SLJIT_PROT_WX; + sljit_uw* ptr; + + if (SLJIT_UNLIKELY(se_protected < 0)) + return NULL; + + size += sizeof(sljit_uw); + ptr = (sljit_uw*)mmap(NULL, size, PROT_READ | PROT_WRITE | SLJIT_PROT_WX, + MAP_PRIVATE | MAP_ANON, -1, 0); + + if (ptr == MAP_FAILED) + return NULL; + + if (SLJIT_UNLIKELY(se_protected > 0)) { + SLJIT_SE_LOCK(); + se_protected = check_se_protected(ptr, size); + SLJIT_SE_UNLOCK(); + if (SLJIT_UNLIKELY(se_protected < 0)) { + munmap((void *)ptr, size); + return NULL; + } + } + + *ptr++ = size; + return ptr; +} + +#undef SLJIT_PROT_WX +#undef SLJIT_SE_UNLOCK +#undef SLJIT_SE_LOCK + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) +{ + sljit_uw *start_ptr = ((sljit_uw*)ptr) - 1; + munmap((void*)start_ptr, *start_ptr); +} + +static void sljit_update_wx_flags(void *from, void *to, sljit_s32 enable_exec) +{ + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + sljit_uw start = (sljit_uw)from; + sljit_uw end = (sljit_uw)to; + int prot = PROT_READ | (enable_exec ? PROT_EXEC : PROT_WRITE); + + SLJIT_ASSERT(start < end); + + start &= ~page_mask; + end = (end + page_mask) & ~page_mask; + + mprotect((void*)start, end - start, prot); +} + +#else /* windows */ + +SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) +{ + sljit_uw *ptr; + + size += sizeof(sljit_uw); + ptr = (sljit_uw*)VirtualAlloc(NULL, size, + MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + + if (!ptr) + return NULL; + + *ptr++ = size; + + return ptr; +} + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) +{ + sljit_uw start = (sljit_uw)ptr - sizeof(sljit_uw); +#if defined(SLJIT_DEBUG) && SLJIT_DEBUG + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + + SLJIT_ASSERT(!(start & page_mask)); +#endif + VirtualFree((void*)start, 0, MEM_RELEASE); +} + +static void sljit_update_wx_flags(void *from, void *to, sljit_s32 enable_exec) +{ + DWORD oldprot; + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + sljit_uw start = (sljit_uw)from; + sljit_uw end = (sljit_uw)to; + DWORD prot = enable_exec ? PAGE_EXECUTE : PAGE_READWRITE; + + SLJIT_ASSERT(start < end); + + start &= ~page_mask; + end = (end + page_mask) & ~page_mask; + + VirtualProtect((void*)start, end - start, prot, &oldprot); +} + +#endif /* !windows */ + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) +{ + /* This allocator does not keep unused memory for future allocations. */ +} diff --git a/dist2/test-driver b/dist2/test-driver index b8521a48..89dba1e0 100755 --- a/dist2/test-driver +++ b/dist2/test-driver @@ -3,7 +3,7 @@ scriptversion=2018-03-07.03; # UTC -# Copyright (C) 2011-2018 Free Software Foundation, Inc. +# Copyright (C) 2011-2020 Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/dist2/testdata/grepoutput b/dist2/testdata/grepoutput index 2bd69bef..5f3b97cc 100644 --- a/dist2/testdata/grepoutput +++ b/dist2/testdata/grepoutput @@ -949,3 +949,34 @@ RC=0 ---------------------------- Test 126 ----------------------------- ABC RC=0 +---------------------------- Test 127 ----------------------------- +pattern +RC=0 +---------------------------- Test 128 ----------------------------- +pcre2grep: Requested group 1 cannot be captured. +pcre2grep: Use --om-capture to increase the size of the capture vector. +RC=2 +---------------------------- Test 129 ----------------------------- +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the +lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox +RC=0 +---------------------------- Test 130 ----------------------------- +fox +fox +fox +fox +RC=0 +---------------------------- Test 131 ----------------------------- +2 +RC=0 +---------------------------- Test 132 ----------------------------- +match 1: + a +match 2: + b +--- + a +RC=0 +---------------------------- Test 133 ----------------------------- +=AB3CD5= +RC=0 diff --git a/dist2/testdata/grepoutput8 b/dist2/testdata/grepoutput8 index aaed6ae3..3888d9a8 100644 --- a/dist2/testdata/grepoutput8 +++ b/dist2/testdata/grepoutput8 @@ -18,3 +18,17 @@ RC=0 22:6,2 22:8,2 RC=0 +---------------------------- Test U4 ------------------------------ +pcre2grep: pcre2_match() gave error -22 while matching this text: + +Aက€CD Z + +UTF-8 error: isolated byte with 0x80 bit set at offset 4 + +RC=1 +---------------------------- Test U5 ------------------------------ +CD Z +RC=0 +---------------------------- Test U6 ----------------------------- +=ǓǤ= +RC=0 diff --git a/dist2/testdata/grepoutputC b/dist2/testdata/grepoutputC index 60f249f4..87897f05 100644 --- a/dist2/testdata/grepoutputC +++ b/dist2/testdata/grepoutputC @@ -40,3 +40,5 @@ T T T T +0:T:AA +The quick brown diff --git a/dist2/testdata/grepoutputCN b/dist2/testdata/grepoutputCN index 5217b5a1..838bee61 100644 --- a/dist2/testdata/grepoutputCN +++ b/dist2/testdata/grepoutputCN @@ -28,3 +28,5 @@ T T T T +0:T:AA +The quick brown diff --git a/dist2/testdata/grepoutputN b/dist2/testdata/grepoutputN index ba97e90a..caaeb759 100644 --- a/dist2/testdata/grepoutputN +++ b/dist2/testdata/grepoutputN @@ -2,16 +2,20 @@ 1:abc
2:def
---------------------------- Test N2 ------------------------------
1:abc
def
2:ghi -jkl---------------------------- Test N3 ------------------------------
+jkl
+---------------------------- Test N3 ------------------------------
2:def
3: ghi -jkl---------------------------- Test N4 ------------------------------
+jkl
---------------------------- Test N4 ------------------------------
2:ghi -jkl---------------------------- Test N5 ------------------------------
+jkl
+---------------------------- Test N5 ------------------------------
1:abc
2:def
3:ghi -4:jkl---------------------------- Test N6 ------------------------------
+4:jkl +---------------------------- Test N6 ------------------------------
1:abc
2:def
3:ghi -4:jkl---------------------------- Test N7 ------------------------------
-1:abcZERO2:def +4:jkl +---------------------------- Test N7 ------------------------------
+1:abcZERO2:defZERO diff --git a/dist2/testdata/testbtables b/dist2/testdata/testbtables Binary files differnew file mode 100644 index 00000000..b7aeeaf0 --- /dev/null +++ b/dist2/testdata/testbtables diff --git a/dist2/testdata/testinput1 b/dist2/testdata/testinput1 index 3fd5d2a6..d4e42ba8 100644 --- a/dist2/testdata/testinput1 +++ b/dist2/testdata/testinput1 @@ -1,6 +1,10 @@ # This set of tests is for features that are compatible with all versions of # Perl >= 5.10, in non-UTF mode. It should run clean for the 8-bit, 16-bit, and -# 32-bit PCRE libraries, and also using the perltest.pl script. +# 32-bit PCRE libraries, and also using the perltest.sh script. + +# WARNING: Use only / as the pattern delimiter. Although pcre2test supports +# a number of delimiters, all those other than / give problems with the +# perltest.sh script. #forbid_utf #newline_default lf any anycrlf @@ -1957,7 +1961,7 @@ abc\nzzz qqq\nabc\nzzz -"(?>.*/)foo" +/(?>.*\/)foo/ /this/is/a/very/long/line/in/deed/with/very/many/slashes/in/and/foo \= Expect no match /this/is/a/very/long/line/in/deed/with/very/many/slashes/in/it/you/see/ @@ -3831,12 +3835,13 @@ /(?-x: )/x A\x20B -"(?x)(?-x: \s*#\s*)" +/(?x)(?-x: \s*#\s*)/ A # B \= Expect no match # + A s#s B -"(?x-is)(?:(?-ixs) \s*#\s*) include" +/(?x-is)(?:(?-ixs) \s*#\s*) include/ A #include \= Expect no match A#include @@ -5232,11 +5237,11 @@ name)/mark alphabetabcd endingwxyz -"(?>.*)foo" +/(?>.*)foo/ \= Expect no match abcdfooxyz -"(?>.*?)foo" +/(?>.*?)foo/ abcdfooxyz /(?:(a(*PRUNE)b)){0}(?:(?1)|ac)/ @@ -5423,18 +5428,6 @@ name)/mark \= Expect no match a -/(?=a\Kb)ab/ - ab - -/(?!a\Kb)ac/ - ac - -/^abc(?<=b\Kc)d/ - abcd - -/^abc(?<!b\Kq)d/ - abcd - /A(*PRUNE:A)A+(*SKIP:A)(B|Z) | AC/x,mark \= Expect no match AAAC @@ -5659,16 +5652,16 @@ name)/mark \= Expect no match acb -'\A(?:[^\"]++|\"(?:[^\"]*+|\"\")*+\")++' +/\A(?:[^\"]++|\"(?:[^\"]*+|\"\")*+\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED -'\A(?:[^\"]++|\"(?:[^\"]++|\"\")*+\")++' +/\A(?:[^\"]++|\"(?:[^\"]++|\"\")*+\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED -'\A(?:[^\"]++|\"(?:[^\"]++|\"\")++\")++' +/\A(?:[^\"]++|\"(?:[^\"]++|\"\")++\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED -'\A([^\"1]++|[\"2]([^\"3]*+|[\"4][\"5])*+[\"6])++' +/\A([^\"1]++|[\"2]([^\"3]*+|[\"4][\"5])*+[\"6])++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED /^\w+(?>\s*)(?<=\w)/ @@ -5727,9 +5720,9 @@ name)/mark /(\2)(\1)/ -"Z*(|d*){216}" +/Z*(|d*){216}/ -"(?1)(?#?'){8}(a)" +/(?1)(?#?'){8}(a)/ baaaaaaaaac /((((((((((((x))))))))))))\12/ @@ -5741,7 +5734,7 @@ name)/mark /(?1)()((((((\1++))\x85)+)|))/ \x85\x85 -"(?|(\k'Pm')|(?'Pm'))" +/(?|(\k'Pm')|(?'Pm'))/ abcd /(?|(aaa)|(b))\g{1}/ @@ -5833,19 +5826,19 @@ ef) x/x,mark \= Expect no match aab -'(?>ab|abab){1,5}?M' +/(?>ab|abab){1,5}?M/ abababababababababababM -'(?>ab|abab){2}?M' +/(?>ab|abab){2}?M/ abababM -'((?(?=(a))a)+k)' +/((?(?=(a))a)+k)/ bbak -'((?(?=(a))a|)+k)' +/((?(?=(a))a|)+k)/ bbak -'(?(?!(b))a|b)+k' +/(?(?!(b))a|b)+k/ ababbalbbadabak /(?!(b))c|b/ @@ -6343,4 +6336,88 @@ ef) x/x,mark /(?&word)* \. (?<word> \w+ )/xi pokus.hokus +/a(?(?=(*:2)b).)/mark + abc + acb + +/a(?(?!(*:2)b).)/mark + acb + abc + +/(?:a|ab){1}+c/ +\= Expect no match + abc + +/(a|ab){1}+c/ + abc + +/(a+){1}+a/ +\= Expect no match + aaaa + +/(?(DEFINE)(a|ab))(?1){1}+c/ + abc + +/(?:a|(?=b)|.)*\z/ + abc + +/(?:a|(?=b)|.)*/ + abc + +/(?<=a(*SKIP)x)|c/ + abcd + +/(?<=a(*SKIP)x)|d/ + abcd + +/(?<=(?=.(?<=x)))/aftertext + abx + +/(?<=(?=(?<=a)))b/ + ab + +/^(?<A>a)(?(<A>)b)((?<=b).*)$/ + abc + +/^(a\1?){4}$/ + aaaa + aaaaaa + +/^((\1+)|\d)+133X$/ + 111133X + +/^(?=.*(?=(([A-Z]).*(?(1)\1)))(?!.+\2)){26}/i + The quick brown fox jumps over the lazy dog. + Jackdaws love my big sphinx of quartz. + Pack my box with five dozen liquor jugs. +\= Expect no match + The quick brown fox jumps over the lazy cat. + Hackdaws love my big sphinx of quartz. + Pack my fox with five dozen liquor jugs. + +/^(?>.*?([A-Z])(?!.*\1)){26}/i + The quick brown fox jumps over the lazy dog. + Jackdaws love my big sphinx of quartz. + Pack my box with five dozen liquor jugs. +\= Expect no match + The quick brown fox jumps over the lazy cat. + Hackdaws love my big sphinx of quartz. + Pack my fox with five dozen liquor jugs. + +/(?<=X(?(DEFINE)(A)))X(*F)/ +\= Expect no match + AXYZ + +/(?<=X(?(DEFINE)(A)))./ + AXYZ + +/(?<=X(?(DEFINE)(.*))Y)./ + AXYZ + +/(?<=X(?(DEFINE)(Y))(?1))./ + AXYZ + +/(?(DEFINE)(?<foo>bar))(?<![-a-z0-9])word/ + word + # End of testinput1 diff --git a/dist2/testdata/testinput10 b/dist2/testdata/testinput10 index 4399f829..53e37cbc 100644 --- a/dist2/testdata/testinput10 +++ b/dist2/testdata/testinput10 @@ -1,7 +1,7 @@ # This set of tests is for UTF-8 support and Unicode property support, with # relevance only for the 8-bit library. -# The next 4 patterns have UTF-8 errors +# The next 5 patterns have UTF-8 errors /[Ã]/utf @@ -11,6 +11,8 @@ /‚‚‚‚‚‚‚Ã/utf +/‚‚‚‚‚‚‚Ã/match_invalid_utf + # Now test subjects /badutf/utf @@ -493,4 +495,126 @@ /(?(á/utf +# Invalid UTF-8 tests + +/.../g,match_invalid_utf + abcd\x80wxzy\x80pqrs + abcd\x{80}wxzy\x80pqrs + +/abc/match_invalid_utf + ab\x80ab\=ph +\= Expect no match + ab\x80cdef\=ph + +/ab$/match_invalid_utf + ab\x80cdeab +\= Expect no match + ab\x80cde + +/.../g,match_invalid_utf + abcd\x{80}wxzy\x80pqrs + +/(?<=x)../g,match_invalid_utf + abcd\x{80}wxzy\x80pqrs + abcd\x{80}wxzy\x80xpqrs + +/X$/match_invalid_utf +\= Expect no match + X\xc4 + +/(?<=..)X/match_invalid_utf,aftertext + AB\x80AQXYZ + AB\x80AQXYZ\=offset=5 + AB\x80\x80AXYZXC\=offset=5 +\= Expect no match + AB\x80XYZ + AB\x80XYZ\=offset=3 + AB\xfeXYZ + AB\xffXYZ\=offset=3 + AB\x80AXYZ + AB\x80AXYZ\=offset=4 + AB\x80\x80AXYZ\=offset=5 + +/.../match_invalid_utf + AB\xc4CCC +\= Expect no match + A\x{d800}B + A\x{110000}B + A\xc4B + +/\bX/match_invalid_utf + A\x80X + +/\BX/match_invalid_utf +\= Expect no match + A\x80X + +/(?<=...)X/match_invalid_utf + AAA\x80BBBXYZ +\= Expect no match + AAA\x80BXYZ + AAA\x80BBXYZ + +# ------------------------------------- + +/(*UTF)(?=\x{123})/I + +/[\x{c1}\x{e1}]X[\x{145}\x{146}]/I,utf + +/[󿾟,]/BI,utf + +/[\x{fff4}-\x{ffff8}]/I,utf + +/[\x{fff4}-\x{afff8}\x{10ffff}]/I,utf + +/[\xff\x{ffff}]/I,utf + +/[\xff\x{ff}]/I,utf + abc\x{ff}def + +/[\xff\x{ff}]/I + abc\x{ff}def + +/[Ss]/I + +/[Ss]/I,utf + +/(?:\x{ff}|\x{3000})/I,utf + +/x/utf + abxyz + \x80\=startchar + abc\x80\=startchar + abc\x80\=startchar,offset=3 + +/\x{c1}+\x{e1}/iIB,ucp + \x{c1}\x{c1}\x{c1} + \x{e1}\x{e1}\x{e1} + +/a|\x{c1}/iI,ucp + \x{e1}xxx + +/a|\x{c1}/iI,utf + \x{e1}xxx + +/\x{c1}|\x{e1}/iI,ucp + +/X(\x{e1})Y/ucp,replace=>\U$1<,substitute_extended + X\x{e1}Y + +/X(\x{e1})Y/i,ucp,replace=>\L$1<,substitute_extended + X\x{c1}Y + +# Without UTF or UCP characters > 127 have only one case in the default locale. + +/X(\x{e1})Y/replace=>\U$1<,substitute_extended + X\x{e1}Y + +/A/utf,match_invalid_utf,caseless + \xe5A + +/\bch\b/utf,match_invalid_utf + qchq\=ph + qchq\=ps + # End of testinput10 diff --git a/dist2/testdata/testinput12 b/dist2/testdata/testinput12 index 466fb93c..9b4f8d34 100644 --- a/dist2/testdata/testinput12 +++ b/dist2/testdata/testinput12 @@ -402,4 +402,148 @@ /(?(á/utf +# Invalid UTF-16/32 tests. + +/.../g,match_invalid_utf + abcd\x{df00}wxzy\x{df00}pqrs + abcd\x{80}wxzy\x{df00}pqrs + +/abc/match_invalid_utf + ab\x{df00}ab\=ph +\= Expect no match + ab\x{df00}cdef\=ph + +/ab$/match_invalid_utf + ab\x{df00}cdeab +\= Expect no match + ab\x{df00}cde + +/.../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + +/(?<=x)../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + abcd\x{80}wxzy\x{df00}xpqrs + +/X$/match_invalid_utf +\= Expect no match + X\x{df00} + +/(?<=..)X/match_invalid_utf,aftertext + AB\x{df00}AQXYZ + AB\x{df00}AQXYZ\=offset=5 + AB\x{df00}\x{df00}AXYZXC\=offset=5 +\= Expect no match + AB\x{df00}XYZ + AB\x{df00}XYZ\=offset=3 + AB\x{df00}AXYZ + AB\x{df00}AXYZ\=offset=4 + AB\x{df00}\x{df00}AXYZ\=offset=5 + +/.../match_invalid_utf +\= Expect no match + A\x{d800}B + A\x{110000}B + +/aa/utf,ucp,match_invalid_utf,global + aa\x{d800}aa + +/aa/utf,ucp,match_invalid_utf,global + \x{d800}aa + +# ---------------------------------------------------- + +/(*UTF)(?=\x{123})/I + +/[\x{c1}\x{e1}]X[\x{145}\x{146}]/I,utf + +/[\xff\x{ffff}]/I,utf + +/[\xff\x{ff}]/I,utf + +/[\xff\x{ff}]/I + +/[Ss]/I + +/[Ss]/I,utf + +/(?:\x{ff}|\x{3000})/I,utf + +# ---------------------------------------------------- +# UCP and casing tests + +/\x{120}/i,I + +/\x{c1}/i,I,ucp + +/[\x{120}\x{121}]/iB,ucp + +/[ab\x{120}]+/iB,ucp + aABb\x{121}\x{120} + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + +/[^\x{120}]/i,no_start_optimize + \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} + +/[^\x{120}]/i + \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} + +/\x{120}{2}/i,ucp + \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} + +/\x{c1}+\x{e1}/iB,ucp + \x{c1}\x{c1}\x{c1} + +/\x{c1}+\x{e1}/iIB,ucp + \x{c1}\x{c1}\x{c1} + \x{e1}\x{e1}\x{e1} + +/a|\x{c1}/iI,ucp + \x{e1}xxx + +/\x{c1}|\x{e1}/iI,ucp + +/X(\x{e1})Y/ucp,replace=>\U$1<,substitute_extended + X\x{e1}Y + +/X(\x{121})Y/ucp,replace=>\U$1<,substitute_extended + X\x{121}Y + +/s/i,ucp + \x{17f} + +/s/i,utf + \x{17f} + +/[^s]/i,ucp +\= Expect no match + \x{17f} + +/[^s]/i,utf +\= Expect no match + \x{17f} + +# ---------------------------------------------------- + # End of testinput12 diff --git a/dist2/testdata/testinput14 b/dist2/testdata/testinput14 index f97f3ec2..8a17ae73 100644 --- a/dist2/testdata/testinput14 +++ b/dist2/testdata/testinput14 @@ -1,9 +1,12 @@ -# These test special (mostly error) UTF features of DFA matching. They are a -# selection of the more comprehensive tests that are run for non-DFA matching. -# The output is different for the different widths. +# These test special UTF and UCP features of DFA matching. The output is +# different for the different widths. #subject dfa +# ---------------------------------------------------- +# These are a selection of the more comprehensive tests that are run for +# non-DFA matching. + /X/utf XX\x{d800} XX\x{d800}\=offset=3 @@ -33,5 +36,46 @@ XX\xef\x80\=ph \xf7\=ph \xf7\x80\=ph + +# ---------------------------------------------------- +# UCP and casing tests - except for the first two, these will all fail in 8-bit +# mode because they are testing UCP without UTF and use characters > 255. + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} + +/\x{c1}+\x{e1}/iB,ucp + \x{c1}\x{c1}\x{c1} + \x{e1}\x{e1}\x{e1} + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + +/[^\x{120}]/i,no_start_optimize + \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} + +/[^\x{120}]/i + \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} + +/\x{120}{2}/i,ucp + \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} + +# ---------------------------------------------------- # End of testinput14 diff --git a/dist2/testdata/testinput15 b/dist2/testdata/testinput15 index 2ef6672a..5dd68979 100644 --- a/dist2/testdata/testinput15 +++ b/dist2/testdata/testinput15 @@ -16,14 +16,14 @@ /^(?>a)++/ aa\=find_limits aaaaaaaaa\=find_limits - + /(a)(?1)++/ aa\=find_limits aaaaaaaaa\=find_limits /a(?:.)*?a/ims abbbbbbbbbbbbbbbbbbbbba\=find_limits - + /a(?:.(*THEN))*?a/ims abbbbbbbbbbbbbbbbbbbbba\=find_limits @@ -86,9 +86,9 @@ aaaaaaaaaaaaaz \= Expect limit exceeded aaaaaaaaaaaaaz\=depth_limit=10 - -# These three have infinitely nested recursions. - + +# These three have infinitely nested recursions. + /((?2))((?1))/ abc @@ -97,21 +97,21 @@ /(?(R)a*(?1)|((?R))b)/ aaaabcde - + # The allusedtext modifier does not work with JIT, which does not maintain # the leftchar/rightchar data. /abc(?=xyz)/allusedtext abcxyzpqr abcxyzpqr\=aftertext - + /(?<=pqr)abc(?=xyz)/allusedtext xyzpqrabcxyzpqr xyzpqrabcxyzpqr\=aftertext - + /a\b/ a.\=allusedtext - a\=allusedtext + a\=allusedtext /abc\Kxyz/ abcxyz\=allusedtext @@ -121,7 +121,57 @@ /abc(?=abcde)(?=ab)/allusedtext abcabcdefg - + +#subject allusedtext + +/(?<=abc)123/ + xyzabc123pqr + xyzabc12\=ps + xyzabc12\=ph + +/\babc\b/ + +++abc+++ + +++ab\=ps + +++ab\=ph + +/(?<=abc)def/ + abc\=ph + +/(?<=123)(*MARK:xx)abc/mark + xxxx123a\=ph + xxxx123a\=ps + +/(?<=(?<=a)b)c.*/I + abc\=ph +\= Expect no match + xbc\=ph + +/(?<=ab)c.*/I + abc\=ph +\= Expect no match + xbc\=ph + +/abc(?<=bc)def/ + xxxabcd\=ph + +/(?<=ab)cdef/ + xxabcd\=ph + +/(?<=(?<=(?<=a)b)c)./I + 123abcXYZ + +/(?<=ab(cd(?<=...)))./I + abcdX + +/(?<=ab((?<=...)cd))./I + ZabcdX + +/(?<=((?<=(?<=ab).))(?1)(?1))./I + abxZ + +#subject +# ------------------------------------------------------------------- + # These tests provoke recursion loops, which give a different error message # when JIT is used. @@ -130,26 +180,26 @@ /(a|(?R))/I abcd - defg + defg /(ab|(bc|(de|(?R))))/I abcd - fghi + fghi /(ab|(bc|(de|(?1))))/I abcd - fghi + fghi /x(ab|(bc|(de|(?1)x)x)x)/I xab123 - xfghi + xfghi /(?!\w)(?R)/ abcd - =abc + =abc /(?=\w)(?R)/ - =abc + =abc abcd /(?<!\w)(?R)/ @@ -160,12 +210,12 @@ /(a+|(?R)b)/ aaa - bbb + bbb /[^\xff]((?1))/BI abcd - -# These tests don't behave the same with JIT + +# These tests don't behave the same with JIT /\w+(?C1)/BI,no_auto_possess abc\=callout_fail=1 @@ -173,7 +223,7 @@ /(*NO_AUTO_POSSESS)\w+(?C1)/BI abc\=callout_fail=1 -# This test breaks the JIT stack limit +# This test breaks the JIT stack limit /(|]+){2,2452}/ (|]+){2,2452} @@ -181,4 +231,8 @@ /(*LIMIT_HEAP=21)\[(a)]{60}/expand \[a]{60} +/b(?<!ax)(?!cx)/allusedtext + abc + abcz + # End of testinput15 diff --git a/dist2/testdata/testinput2 b/dist2/testdata/testinput2 index 9e59b628..3f9dd6d9 100644 --- a/dist2/testdata/testinput2 +++ b/dist2/testdata/testinput2 @@ -324,16 +324,7 @@ \= Expect no match fooabar -# This one is here because Perl behaves differently; see also the following. - -/^(a\1?){4}$/I -\= Expect no match - aaaa - aaaaaa - -# Perl does not fail these two for the final subjects. Neither did PCRE until -# release 8.01. The problem is in backtracking into a subpattern that contains -# a recursive reference to itself. PCRE has now made these into atomic patterns. +# Perl does not fail these two for the final subjects. /^(xa|=?\1a){2}$/ xa=xaa @@ -1625,6 +1616,7 @@ /^a*A\d/IBi aaaA5 aaaa5 + a5 /(a*|b*)[cd]/I @@ -4584,13 +4576,19 @@ B)x/alt_verbnames,mark /abcd/null_context abcd\=null_context -\= Expect error +\= Expect error - not allowed together abcd\=null_context,find_limits abcd\=allusedtext,startchar /abcd/replace=w\rx\x82y\o{333}z(\Q12\$34$$\x34\E5$$),substitute_extended abcd +/abcd/replace=w\rx\x82y\o{333}z(\Q12\$34$$\x34\E5$$),substitute_extended,substitute_literal + >>abcd<< + +/abcd/g,replace=\$1$2\,substitute_literal + XabcdYabcdZ + /a(bc)(DE)/replace=a\u$1\U$1\E$1\l$2\L$2\Eab\Uab\LYZ\EDone,substitute_extended abcDE @@ -4602,6 +4600,7 @@ B)x/alt_verbnames,mark ac ab\=replace=${1:+$1\:$1:$2} ac\=replace=${1:+$1\:$1:$2} + >>ac<<\=replace=${1:+$1\:$1:$2},substitute_literal /a(?:(b)|(c))/substitute_extended,replace=X${1:-1:-1}X${2:-2:-2} ab @@ -4641,6 +4640,13 @@ B)x/alt_verbnames,mark /(aa)(BB)/substitute_extended,replace=\U$1\L$2\E$1..\U$1\l$2$1 aaBB + +/abcd/replace=wxyz,substitute_matched + abcd + pqrs + +/abcd/g + >abcd1234abcd5678<\=replace=wxyz,substitute_matched /^(o(\1{72}{\"{\\{00000059079}\d*){74}}){19}/I @@ -5587,4 +5593,293 @@ a)"xI \= Expect error message abc\=null_context +/\[()]{65535}()/expand + +/\[()]{65535}(?<A>)/expand + +/a(?:(*ACCEPT))??bc/ + abc + axy + +/a(*ACCEPT)??bc/ + abc + axy + +/a(*ACCEPT:XX)??bc/mark + abc + axy + +/(*:\)?/ + +/(*:\Q \E){5}/alt_verbnames + +/(?=abc)/I + +/(?|(X)|(XY))\1abc/I + +/(?|(a)|(bcde))(c)\2/I + +/(?|(a)|(bcde))(c)\1/I + +/(?|(?'A'a)|(?'A'bcde))(?'B'c)\k'B'(?'A')/I,dupnames + +/(?|(?'A'a)|(?'A'bcde))(?'B'c)\k'A'(?'A')/I,dupnames + +/((a|)+)+Z/I + +/((?=a))[abcd]/I + +/A(?:(*ACCEPT))?B/info + +/(A(*ACCEPT)??B)C/ + ABC + AXY + +/(?<=(?<=a)b)c.*/I + abc\=ph +\= Expect no match + xbc\=ph + +/(?<=ab)c.*/I + abc\=ph +\= Expect no match + xbc\=ph + +/(?<=a(?<=a|a)c)/I + +/(?<=a(?<=a|ba)c)/I + +/(?<=(?<=a)b)(?<!abcd)/I + +/(?<=(?<=a)b)(?<!abcd)(?<=(?<=a)bcde)/I + +# Addition overflow +/( {32742} {42})(?<!\1{65481})/ + +# Multiplication overflow +/(X{65535})(?<=\1{32770})/ + +# ---- Non-atomic assertion tests ---- + +# Expect error: not allowed as a condition +/(?(*napla:xx)bc)/ + +/\A(*pla:.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 + +/\A(*napla:.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 + +/\A(?*.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 + +/(*plb:(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + abcda\=offset=4 + +/(*naplb:(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + abcda\=offset=4 + +/(?<*(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + abcda\=offset=4 + +/(*non_atomic_positive_lookahead:ab)/B + +/(*non_atomic_positive_lookbehind:ab)/B + +/(*pla:ab+)/B + +/(*napla:ab+)/B + +/(*napla:)+/ + +/(*naplb:)+/ + +/(*napla:^x|^y)/I + +/(*napla:abc|abd)/I + +/(*napla:a|(.)(*ACCEPT)zz)\1../ + abcd + +/(*napla:a(*ACCEPT)zz|(.))\1../ + abcd + +/(*napla:a|(*COMMIT)(.))\1\1/ + aabc +\= Expect no match + abbc + +/(*napla:a|(.))\1\1/ + aabc + abbc + +# ---- + +# Expect error (recursion => not fixed length) +/(\2)((?=(?<=\1)))/ + +/c*+(?<=[bc])/ + abc\=ph + ab\=ph + abc\=ps + ab\=ps + +/c++(?<=[bc])/ + abc\=ph + ab\=ph + +/(?<=(?=.(?<=x)))/ + abx + ab\=ph + bxyz + xyz + +/\z/ + abc\=ph + abc\=ps + +/\Z/ + abc\=ph + abc\=ps + abc\n\=ph + abc\n\=ps + +/(?![ab]).*/ + ab\=ph + +/c*+/ + ab\=ph,offset=2 + +/\A\s*(a|(?:[^`]{28500}){4})/I + a + +/\A\s*((?:[^`]{28500}){4})/I + +/\A\s*((?:[^`]{28500}){4}|a)/I + a + +/(?<A>a)(?(<A>)b)((?<=b).*)/B + +/(?(1)b)((?<=b).*)/B + +/(?(R1)b)((?<=b).*)/B + +/(?(DEFINE)b)((?<=b).*)/B + +/(?(VERSION=10.4)b)((?<=b).*)/B + +/[aA]b[cC]/IB + +/[cc]abcd/I + +/[Cc]abcd/I + +/[c]abcd/I + +/(?:c|C)abcd/I + +/(a)?a/I + manm + +/^(?|(\*)(*napla:\S*_(\2?+.+))|(\w)(?=\S*_(\2?+\1)))+_\2$/ + *abc_12345abc + +/^(?|(\*)(*napla:\S*_(\3?+.+))|(\w)(?=\S*_((\2?+\1))))+_\2$/ + *abc_12345abc + +/^((\1+)(?C)|\d)+133X$/ + 111133X\=callout_capture + +/abc/replace=xyz,substitute_replacement_only + 123abc456 + +/a(?<ONE>b)c(?<TWO>d)e/g,replace=X$ONE+${TWO}Z,substitute_replacement_only + "abcde-abcde-" + +/a(b)c|xyz/g,replace=<$0>,substitute_callout,substitute_replacement_only + abcdefabcpqr + abxyzpqrabcxyz + 12abc34xyz99abc55\=substitute_stop=2 + 12abc34xyz99abc55\=substitute_skip=1 + 12abc34xyz99abc55\=substitute_skip=2 + +/a(..)d/replace=>$1<,substitute_matched + xyzabcdxyzabcdxyz + xyzabcdxyzabcdxyz\=ovector=2 +\= Expect error + xyzabcdxyzabcdxyz\=ovector=1 + +/a(..)d/g,replace=>$1<,substitute_matched + xyzabcdxyzabcdxyz + xyzabcdxyzabcdxyz\=ovector=2 +\= Expect error + xyzabcdxyzabcdxyz\=ovector=1 + xyzabcdxyzabcdxyz\=ovector=1,substitute_unset_empty + +/55|a(..)d/g,replace=>$1<,substitute_matched + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty +\= Expect error + xyz55abcdxyzabcdxyz\=ovector=2 + +/55|a(..)d/replace=>$1<,substitute_matched + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + +/55|a(..)d/replace=>$1< + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + +/55|a(..)d/g,replace=>$1< + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + +# Expect non-fixed-length error + +"(?<=X(?(DEFINE)(.*))(?1))." + +/\sxxx\s/tables=1 +\= Expect no match + AB\x{85}xxx\x{a0}XYZ + +/\sxxx\s/tables=2 + AB\x{85}xxx\x{a0}XYZ + +/^\w+/tables=2 + École + +/^\w+/tables=3 + École + +#loadtables ./testbtables + +/^\w+/tables=3 + École + +/"(*MARK:>" 00 "<).."/hex,mark,no_start_optimize + AB + A\=ph +\= Expect no match + A + +/"(*MARK:>" 00 "<).(?C1)."/hex,mark,no_start_optimize + AB + +/(?(VERSION=0.0/ + +# Perl has made \K in lookarounds an error. At the moment PCRE2 still accepts. + +/(?=a\Kb)ab/ + ab + +/(?!a\Kb)ac/ + ac + +/^abc(?<=b\Kc)d/ + abcd + +/^abc(?<!b\Kq)d/ + abcd + +# --------- + # End of testinput2 diff --git a/dist2/testdata/testinput4 b/dist2/testdata/testinput4 index cccab0e5..4e2a0abc 100644 --- a/dist2/testdata/testinput4 +++ b/dist2/testdata/testinput4 @@ -3,6 +3,10 @@ # some of the property tests may differ because of different versions of # Unicode in use by PCRE2 and Perl. +# WARNING: Use only / as the pattern delimiter. Although pcre2test supports +# a number of delimiters, all those other than / give problems with the +# perltest.sh script. + #newline_default lf anycrlf any #perltest @@ -476,7 +480,7 @@ /^\ሴ/utf ሴ -"(?s)(.{1,5})"utf +/(?s)(.{1,5})/utf abcdefg ab @@ -804,10 +808,10 @@ \x{4d00} \x{4db4} \x{4db5} + \x{4db6} \= Expect no match a \x{2b0} - \x{4db6} /^\p{Lt}/utf \x{1c5} @@ -2220,7 +2224,7 @@ /[A-`]/i,utf abcdefghijklmno -"[\S\V\H]"utf +/[\S\V\H]/utf /[^\p{Any}]*+x/utf x @@ -2480,4 +2484,15 @@ /^(?'×ABC'...)(?&×ABC)/utf 123123123456 +/\X*/ + \xF3aaa\xE4\xEA\xEB\xFEa + +/Я/i,utf + \x{42f} + \x{44f} + +/(?=Я)/i,utf + \x{42f} + \x{44f} + # End of testinput4 diff --git a/dist2/testdata/testinput5 b/dist2/testdata/testinput5 index 7c581450..50dfda14 100644 --- a/dist2/testdata/testinput5 +++ b/dist2/testdata/testinput5 @@ -2081,7 +2081,6 @@ \x{655} /^\p{Common}/utf - \x{589} \x{60c} \x{61f} \x{964} @@ -2152,6 +2151,16 @@ A\x{1d7ce}\x{1d7cf}B Latin fancy-common-digits Latin \x{1d7ce}\x{1d7cf},, fancy-common-digits Common Common A\x{1d7ce}BC Latin fancy-common-digit Latin Latin + +# Some Unicode 12.1.0 new script characters + +/\p{Elymaic}\p{Nandinagari}\p{Nyiakeng_Puachue_Hmong}\p{Wancho}/utf + \x{10fe5}\x{119AC}\x{1E10E}\x{1E2D1} + +# Some Unicode 13.0.0 new script characters + +/\p{Chorasmian}\p{Dives_Akuru}\p{Khitan_Small_Script}\p{Yezidi}/utf + \x{10FB0}\x{11900}\x{18B00}\x{10E80} # ------- @@ -2174,4 +2183,9 @@ /\p{Any}*xyz/I +/(|ß)7/caseless,ucp + +/(\xc1)\1/i,ucp + \xc1\xe1\=no_jit + # End of testinput5 diff --git a/dist2/testdata/testinput6 b/dist2/testdata/testinput6 index 15dfb74b..0ca0d23c 100644 --- a/dist2/testdata/testinput6 +++ b/dist2/testdata/testinput6 @@ -486,7 +486,7 @@ def\=dfa_restart /(?<=foo)bar/ - foob\=ps,offset=2 + foob\=ps,offset=2,allusedtext foobar...\=ps,dfa_restart,offset=4 foobar\=offset=2 \= Expect no match @@ -4415,12 +4415,12 @@ /abc\K123/ xyzabc123pqr -/(?<=abc)123/ +/(?<=abc)123/allusedtext xyzabc123pqr xyzabc12\=ps xyzabc12\=ph -/\babc\b/ +/\babc\b/allusedtext +++abc+++ +++ab\=ps +++ab\=ph @@ -4490,7 +4490,7 @@ /^(?(?!a(*SKIP)b))/ ac -/(?<=abc)def/ +/(?<=abc)def/allusedtext abc\=ph /abc$/ @@ -4972,4 +4972,58 @@ \= Expect no match 0 +/(?<=pqr)abc(?=xyz)/ + 123pqrabcxy\=ps,allusedtext + 123pqrabcxyz\=ps,allusedtext + +/(?>a+b)/ + aaaa\=ps + aaaab\=ps + +/(abc)(?1)/ + abca\=ps + abcabc\=ps + +/(?(?=abc).*|Z)/ + ab\=ps + abcxyz\=ps + +/(abc)++x/ + abcab\=ps + abc\=ps + ab\=ps + abcx + +/\z/ + abc\=ph + abc\=ps + +/\Z/ + abc\=ph + abc\=ps + abc\n\=ph + abc\n\=ps + +/c*+(?<=[bc])/ + abc\=ph + ab\=ph + abc\=ps + ab\=ps + +/c++(?<=[bc])/ + abc\=ph + ab\=ph + +/(?<=(?=.(?<=x)))/ + abx + ab\=ph + bxyz + xyz + +/(?![ab]).*/ + ab\=ph + +/c*+/ + ab\=ph,offset=2 + # End of testinput6 diff --git a/dist2/testdata/testinput8 b/dist2/testdata/testinput8 index 2627454f..550631db 100644 --- a/dist2/testdata/testinput8 +++ b/dist2/testdata/testinput8 @@ -182,4 +182,8 @@ /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput1 b/dist2/testdata/testoutput1 index 8fd67b34..16c91d2b 100644 --- a/dist2/testdata/testoutput1 +++ b/dist2/testdata/testoutput1 @@ -1,6 +1,10 @@ # This set of tests is for features that are compatible with all versions of # Perl >= 5.10, in non-UTF mode. It should run clean for the 8-bit, 16-bit, and -# 32-bit PCRE libraries, and also using the perltest.pl script. +# 32-bit PCRE libraries, and also using the perltest.sh script. + +# WARNING: Use only / as the pattern delimiter. Although pcre2test supports +# a number of delimiters, all those other than / give problems with the +# perltest.sh script. #forbid_utf #newline_default lf any anycrlf @@ -2949,7 +2953,7 @@ No match qqq\nabc\nzzz No match -"(?>.*/)foo" +/(?>.*\/)foo/ /this/is/a/very/long/line/in/deed/with/very/many/slashes/in/and/foo 0: /this/is/a/very/long/line/in/deed/with/very/many/slashes/in/and/foo \= Expect no match @@ -6081,14 +6085,16 @@ No match A\x20B 0: -"(?x)(?-x: \s*#\s*)" +/(?x)(?-x: \s*#\s*)/ A # B 0: # \= Expect no match # No match + A s#s B +No match -"(?x-is)(?:(?-ixs) \s*#\s*) include" +/(?x-is)(?:(?-ixs) \s*#\s*) include/ A #include 0: #include \= Expect no match @@ -8402,12 +8408,12 @@ No match 1: <unset> 2: wxyz -"(?>.*)foo" +/(?>.*)foo/ \= Expect no match abcdfooxyz No match -"(?>.*?)foo" +/(?>.*?)foo/ abcdfooxyz 0: foo @@ -8648,22 +8654,6 @@ No match a No match -/(?=a\Kb)ab/ - ab - 0: b - -/(?!a\Kb)ac/ - ac - 0: ac - -/^abc(?<=b\Kc)d/ - abcd - 0: cd - -/^abc(?<!b\Kq)d/ - abcd - 0: abcd - /A(*PRUNE:A)A+(*SKIP:A)(B|Z) | AC/x,mark \= Expect no match AAAC @@ -8988,19 +8978,19 @@ No match acb No match -'\A(?:[^\"]++|\"(?:[^\"]*+|\"\")*+\")++' +/\A(?:[^\"]++|\"(?:[^\"]*+|\"\")*+\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED 0: NON QUOTED "QUOT""ED" AFTER -'\A(?:[^\"]++|\"(?:[^\"]++|\"\")*+\")++' +/\A(?:[^\"]++|\"(?:[^\"]++|\"\")*+\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED 0: NON QUOTED "QUOT""ED" AFTER -'\A(?:[^\"]++|\"(?:[^\"]++|\"\")++\")++' +/\A(?:[^\"]++|\"(?:[^\"]++|\"\")++\")++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED 0: NON QUOTED "QUOT""ED" AFTER -'\A([^\"1]++|[\"2]([^\"3]*+|[\"4][\"5])*+[\"6])++' +/\A([^\"1]++|[\"2]([^\"3]*+|[\"4][\"5])*+[\"6])++/ NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED 0: NON QUOTED "QUOT""ED" AFTER 1: AFTER @@ -9128,9 +9118,9 @@ No match /(\2)(\1)/ -"Z*(|d*){216}" +/Z*(|d*){216}/ -"(?1)(?#?'){8}(a)" +/(?1)(?#?'){8}(a)/ baaaaaaaaac 0: aaaaaaaaa 1: a @@ -9166,7 +9156,7 @@ No match 6: 7: -"(?|(\k'Pm')|(?'Pm'))" +/(?|(\k'Pm')|(?'Pm'))/ abcd 0: 1: @@ -9322,27 +9312,27 @@ No match aab No match -'(?>ab|abab){1,5}?M' +/(?>ab|abab){1,5}?M/ abababababababababababM 0: abababababM -'(?>ab|abab){2}?M' +/(?>ab|abab){2}?M/ abababM 0: ababM -'((?(?=(a))a)+k)' +/((?(?=(a))a)+k)/ bbak 0: ak 1: ak 2: a -'((?(?=(a))a|)+k)' +/((?(?=(a))a|)+k)/ bbak 0: ak 1: ak 2: a -'(?(?!(b))a|b)+k' +/(?(?!(b))a|b)+k/ ababbalbbadabak 0: abak 1: b @@ -10049,4 +10039,141 @@ No match 0: pokus.hokus 1: hokus +/a(?(?=(*:2)b).)/mark + abc + 0: ab +MK: 2 + acb + 0: a + +/a(?(?!(*:2)b).)/mark + acb + 0: ac + abc + 0: a +MK: 2 + +/(?:a|ab){1}+c/ +\= Expect no match + abc +No match + +/(a|ab){1}+c/ + abc +No match + +/(a+){1}+a/ +\= Expect no match + aaaa +No match + +/(?(DEFINE)(a|ab))(?1){1}+c/ + abc +No match + +/(?:a|(?=b)|.)*\z/ + abc + 0: abc + +/(?:a|(?=b)|.)*/ + abc + 0: a + +/(?<=a(*SKIP)x)|c/ + abcd +No match + +/(?<=a(*SKIP)x)|d/ + abcd + 0: d + +/(?<=(?=.(?<=x)))/aftertext + abx + 0: + 0+ x + +/(?<=(?=(?<=a)))b/ + ab + 0: b + +/^(?<A>a)(?(<A>)b)((?<=b).*)$/ + abc + 0: abc + 1: a + 2: c + +/^(a\1?){4}$/ + aaaa + 0: aaaa + 1: a + aaaaaa + 0: aaaaaa + 1: aa + +/^((\1+)|\d)+133X$/ + 111133X + 0: 111133X + 1: 11 + 2: 11 + +/^(?=.*(?=(([A-Z]).*(?(1)\1)))(?!.+\2)){26}/i + The quick brown fox jumps over the lazy dog. + 0: + 1: quick brown fox jumps over the lazy dog. + 2: q + Jackdaws love my big sphinx of quartz. + 0: + 1: Jackdaws love my big sphinx of quartz. + 2: J + Pack my box with five dozen liquor jugs. + 0: + 1: Pack my box with five dozen liquor jugs. + 2: P +\= Expect no match + The quick brown fox jumps over the lazy cat. +No match + Hackdaws love my big sphinx of quartz. +No match + Pack my fox with five dozen liquor jugs. +No match + +/^(?>.*?([A-Z])(?!.*\1)){26}/i + The quick brown fox jumps over the lazy dog. + 0: The quick brown fox jumps over the lazy dog + 1: g + Jackdaws love my big sphinx of quartz. + 0: Jackdaws love my big sphinx of quartz + 1: z + Pack my box with five dozen liquor jugs. + 0: Pack my box with five dozen liquor jugs + 1: s +\= Expect no match + The quick brown fox jumps over the lazy cat. +No match + Hackdaws love my big sphinx of quartz. +No match + Pack my fox with five dozen liquor jugs. +No match + +/(?<=X(?(DEFINE)(A)))X(*F)/ +\= Expect no match + AXYZ +No match + +/(?<=X(?(DEFINE)(A)))./ + AXYZ + 0: Y + +/(?<=X(?(DEFINE)(.*))Y)./ + AXYZ + 0: Z + +/(?<=X(?(DEFINE)(Y))(?1))./ + AXYZ + 0: Z + +/(?(DEFINE)(?<foo>bar))(?<![-a-z0-9])word/ + word + 0: word + # End of testinput1 diff --git a/dist2/testdata/testoutput10 b/dist2/testdata/testoutput10 index dfecda1c..d4085106 100644 --- a/dist2/testdata/testoutput10 +++ b/dist2/testdata/testoutput10 @@ -1,7 +1,7 @@ # This set of tests is for UTF-8 support and Unicode property support, with # relevance only for the 8-bit library. -# The next 4 patterns have UTF-8 errors +# The next 5 patterns have UTF-8 errors /[Ã]/utf Failed: error -8 at offset 1: UTF-8 error: byte 2 top bits not 0x80 @@ -15,6 +15,9 @@ Failed: error -8 at offset 0: UTF-8 error: byte 2 top bits not 0x80 /‚‚‚‚‚‚‚Ã/utf Failed: error -22 at offset 2: UTF-8 error: isolated byte with 0x80 bit set +/‚‚‚‚‚‚‚Ã/match_invalid_utf +Failed: error -22 at offset 2: UTF-8 error: isolated byte with 0x80 bit set + # Now test subjects /badutf/utf @@ -1253,11 +1256,7 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Capture group count = 0 Options: utf -Starting code units: Z \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd - \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc - \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb - \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 \xf8 \xf9 \xfa - \xfb \xfc \xfd \xfe \xff +Starting code units: Z \xc4 Subject length lower bound = 1 Z\x{100} 0: Z @@ -1275,11 +1274,7 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Capture group count = 0 Options: utf -Starting code units: z { | } ~ \x7f \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 - \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 - \xd9 \xda \xdb \xdc \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 - \xe8 \xe9 \xea \xeb \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 - \xf7 \xf8 \xf9 \xfa \xfb \xfc \xfd \xfe \xff +Starting code units: z { | } ~ \x7f \xc2 \xc3 \xc4 Subject length lower bound = 1 /[z\Qa-d]Ä€\E]/IB,utf @@ -1291,11 +1286,7 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Capture group count = 0 Options: utf -Starting code units: - ] a d z \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc - \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb - \xdc \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea - \xeb \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 \xf8 \xf9 - \xfa \xfb \xfc \xfd \xfe \xff +Starting code units: - ] a d z \xc4 Subject length lower bound = 1 \x{100} 0: \x{100} @@ -1316,11 +1307,7 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Capture group count = 1 Options: utf -Starting code units: a b \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd - \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc - \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb - \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 \xf8 \xf9 \xfa - \xfb \xfc \xfd \xfe \xff +Starting code units: a b \xc4 Last code unit = 'z' Subject length lower bound = 7 @@ -1437,11 +1424,7 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Capture group count = 0 Options: caseless utf -Starting code units: \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce - \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd - \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec - \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 \xf8 \xf9 \xfa \xfb - \xfc \xfd \xfe \xff +Starting code units: \xc4 Subject length lower bound = 1 \x{104} 0: \x{104} @@ -1464,11 +1447,7 @@ No match ------------------------------------------------------------------ Capture group count = 0 Options: caseless utf -Starting code units: Z z { | } ~ \x7f \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 - \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 - \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 - \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 - \xf6 \xf7 \xf8 \xf9 \xfa \xfb \xfc \xfd \xfe \xff +Starting code units: Z z { | } ~ \x7f \xc2 \xc3 \xc4 \xc5 \xce \xe1 \xe2 Subject length lower bound = 1 Z 0: Z @@ -1505,11 +1484,7 @@ No match ------------------------------------------------------------------ Capture group count = 0 Options: caseless utf -Starting code units: Z z { | } ~ \x7f \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 - \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 - \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf \xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 - \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef \xf0 \xf1 \xf2 \xf3 \xf4 \xf5 - \xf6 \xf7 \xf8 \xf9 \xfa \xfb \xfc \xfd \xfe \xff +Starting code units: Z z { | } ~ \x7f \xc2 \xc3 \xc4 \xc5 \xce \xe1 \xe2 Subject length lower bound = 1 /\x{3a3}B/IBi,utf @@ -1651,4 +1626,259 @@ Failed: error 142 at offset 4: syntax error in subpattern name (missing terminat /(?(á/utf Failed: error 142 at offset 5: syntax error in subpattern name (missing terminator?) +# Invalid UTF-8 tests + +/.../g,match_invalid_utf + abcd\x80wxzy\x80pqrs + 0: abc + 0: wxz + 0: pqr + abcd\x{80}wxzy\x80pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/abc/match_invalid_utf + ab\x80ab\=ph +Partial match: ab +\= Expect no match + ab\x80cdef\=ph +No match + +/ab$/match_invalid_utf + ab\x80cdeab + 0: ab +\= Expect no match + ab\x80cde +No match + +/.../g,match_invalid_utf + abcd\x{80}wxzy\x80pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/(?<=x)../g,match_invalid_utf + abcd\x{80}wxzy\x80pqrs + 0: zy + abcd\x{80}wxzy\x80xpqrs + 0: zy + 0: pq + +/X$/match_invalid_utf +\= Expect no match + X\xc4 +No match + +/(?<=..)X/match_invalid_utf,aftertext + AB\x80AQXYZ + 0: X + 0+ YZ + AB\x80AQXYZ\=offset=5 + 0: X + 0+ YZ + AB\x80\x80AXYZXC\=offset=5 + 0: X + 0+ C +\= Expect no match + AB\x80XYZ +No match + AB\x80XYZ\=offset=3 +No match + AB\xfeXYZ +No match + AB\xffXYZ\=offset=3 +No match + AB\x80AXYZ +No match + AB\x80AXYZ\=offset=4 +No match + AB\x80\x80AXYZ\=offset=5 +No match + +/.../match_invalid_utf + AB\xc4CCC + 0: CCC +\= Expect no match + A\x{d800}B +No match + A\x{110000}B +No match + A\xc4B +No match + +/\bX/match_invalid_utf + A\x80X + 0: X + +/\BX/match_invalid_utf +\= Expect no match + A\x80X +No match + +/(?<=...)X/match_invalid_utf + AAA\x80BBBXYZ + 0: X +\= Expect no match + AAA\x80BXYZ +No match + AAA\x80BBXYZ +No match + +# ------------------------------------- + +/(*UTF)(?=\x{123})/I +Capture group count = 0 +May match empty string +Compile options: <none> +Overall options: utf +First code unit = \xc4 +Last code unit = \xa3 +Subject length lower bound = 1 + +/[\x{c1}\x{e1}]X[\x{145}\x{146}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xc3 +Last code unit = 'X' +Subject length lower bound = 3 + +/[󿾟,]/BI,utf +------------------------------------------------------------------ + Bra + [,\x{fff9f}] + Ket + End +------------------------------------------------------------------ +Capture group count = 0 +Options: utf +Starting code units: , \xf3 +Subject length lower bound = 1 + +/[\x{fff4}-\x{ffff8}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xef \xf0 \xf1 \xf2 \xf3 +Subject length lower bound = 1 + +/[\x{fff4}-\x{afff8}\x{10ffff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xef \xf0 \xf1 \xf2 \xf4 +Subject length lower bound = 1 + +/[\xff\x{ffff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xc3 \xef +Subject length lower bound = 1 + +/[\xff\x{ff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xc3 +Subject length lower bound = 1 + abc\x{ff}def + 0: \x{ff} + +/[\xff\x{ff}]/I +Capture group count = 0 +First code unit = \xff +Subject length lower bound = 1 + abc\x{ff}def + 0: \xff + +/[Ss]/I +Capture group count = 0 +First code unit = 'S' (caseless) +Subject length lower bound = 1 + +/[Ss]/I,utf +Capture group count = 0 +Options: utf +Starting code units: S s +Subject length lower bound = 1 + +/(?:\x{ff}|\x{3000})/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xc3 \xe3 +Subject length lower bound = 1 + +/x/utf + abxyz + 0: x + \x80\=startchar +Failed: error -22: UTF-8 error: isolated byte with 0x80 bit set at offset 0 + abc\x80\=startchar +Failed: error -22: UTF-8 error: isolated byte with 0x80 bit set at offset 3 + abc\x80\=startchar,offset=3 +Error -36 (bad UTF-8 offset) + +/\x{c1}+\x{e1}/iIB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Last code unit = \xe1 (caseless) +Subject length lower bound = 2 + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + +/a|\x{c1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +Starting code units: A a \xc1 \xe1 +Subject length lower bound = 1 + \x{e1}xxx + 0: \xe1 + +/a|\x{c1}/iI,utf +Capture group count = 0 +Options: caseless utf +Starting code units: A a \xc3 +Subject length lower bound = 1 + \x{e1}xxx + 0: \x{e1} + +/\x{c1}|\x{e1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Subject length lower bound = 1 + +/X(\x{e1})Y/ucp,replace=>\U$1<,substitute_extended + X\x{e1}Y + 1: >\xc1< + +/X(\x{e1})Y/i,ucp,replace=>\L$1<,substitute_extended + X\x{c1}Y + 1: >\xe1< + +# Without UTF or UCP characters > 127 have only one case in the default locale. + +/X(\x{e1})Y/replace=>\U$1<,substitute_extended + X\x{e1}Y + 1: >\xe1< + +/A/utf,match_invalid_utf,caseless + \xe5A + 0: A + +/\bch\b/utf,match_invalid_utf + qchq\=ph +Partial match: + qchq\=ps +Partial match: + # End of testinput10 diff --git a/dist2/testdata/testoutput12-16 b/dist2/testdata/testoutput12-16 index 3f2c91fe..84c48581 100644 --- a/dist2/testdata/testoutput12-16 +++ b/dist2/testdata/testoutput12-16 @@ -533,7 +533,7 @@ Failed: error -26: UTF-16 error: isolated low surrogate at offset 2 XX\x{110000} ** Failed: character \x{110000} is greater than 0x10ffff and so cannot be converted to UTF-16 XX\x{d800}\x{1234} -Failed: error -25: UTF-16 error: invalid low surrogate at offset 3 +Failed: error -25: UTF-16 error: invalid low surrogate at offset 2 \= Expect no match XX\x{d800}\=offset=3 No match @@ -1502,4 +1502,283 @@ Failed: error 142 at offset 4: syntax error in subpattern name (missing terminat /(?(á/utf Failed: error 142 at offset 4: syntax error in subpattern name (missing terminator?) +# Invalid UTF-16/32 tests. + +/.../g,match_invalid_utf + abcd\x{df00}wxzy\x{df00}pqrs + 0: abc + 0: wxz + 0: pqr + abcd\x{80}wxzy\x{df00}pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/abc/match_invalid_utf + ab\x{df00}ab\=ph +Partial match: ab +\= Expect no match + ab\x{df00}cdef\=ph +No match + +/ab$/match_invalid_utf + ab\x{df00}cdeab + 0: ab +\= Expect no match + ab\x{df00}cde +No match + +/.../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/(?<=x)../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + 0: zy + abcd\x{80}wxzy\x{df00}xpqrs + 0: zy + 0: pq + +/X$/match_invalid_utf +\= Expect no match + X\x{df00} +No match + +/(?<=..)X/match_invalid_utf,aftertext + AB\x{df00}AQXYZ + 0: X + 0+ YZ + AB\x{df00}AQXYZ\=offset=5 + 0: X + 0+ YZ + AB\x{df00}\x{df00}AXYZXC\=offset=5 + 0: X + 0+ C +\= Expect no match + AB\x{df00}XYZ +No match + AB\x{df00}XYZ\=offset=3 +No match + AB\x{df00}AXYZ +No match + AB\x{df00}AXYZ\=offset=4 +No match + AB\x{df00}\x{df00}AXYZ\=offset=5 +No match + +/.../match_invalid_utf +\= Expect no match + A\x{d800}B +No match + A\x{110000}B +** Failed: character \x{110000} is greater than 0x10ffff and so cannot be converted to UTF-16 + +/aa/utf,ucp,match_invalid_utf,global + aa\x{d800}aa + 0: aa + 0: aa + +/aa/utf,ucp,match_invalid_utf,global + \x{d800}aa + 0: aa + +# ---------------------------------------------------- + +/(*UTF)(?=\x{123})/I +Capture group count = 0 +May match empty string +Compile options: <none> +Overall options: utf +First code unit = \x{123} +Subject length lower bound = 1 + +/[\x{c1}\x{e1}]X[\x{145}\x{146}]/I,utf +Capture group count = 0 +Options: utf +First code unit = \xc1 (caseless) +Last code unit = \x{145} (caseless) +Subject length lower bound = 3 + +/[\xff\x{ffff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +/[\xff\x{ff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +/[\xff\x{ff}]/I +Capture group count = 0 +Starting code units: \xff +Subject length lower bound = 1 + +/[Ss]/I +Capture group count = 0 +First code unit = 'S' (caseless) +Subject length lower bound = 1 + +/[Ss]/I,utf +Capture group count = 0 +Options: utf +Starting code units: S s +Subject length lower bound = 1 + +/(?:\x{ff}|\x{3000})/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +# ---------------------------------------------------- +# UCP and casing tests + +/\x{120}/i,I +Capture group count = 0 +Options: caseless +First code unit = \x{120} +Subject length lower bound = 1 + +/\x{c1}/i,I,ucp +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Subject length lower bound = 1 + +/[\x{120}\x{121}]/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{120} + Ket + End +------------------------------------------------------------------ + +/[ab\x{120}]+/iB,ucp +------------------------------------------------------------------ + Bra + [ABab\x{120}-\x{121}]++ + Ket + End +------------------------------------------------------------------ + aABb\x{121}\x{120} + 0: aABb\x{121}\x{120} + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} +No match + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + 0: \x{121}\xe1 + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + 0: \x{121}\xe1 + +/[^\x{120}]/i,no_start_optimize + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} +No match + +/[^\x{120}]/i + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} +No match + +/\x{120}{2}/i,ucp + \x{121}\x{121} + 0: \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} +No match + +/\x{c1}+\x{e1}/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + +/\x{c1}+\x{e1}/iIB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Last code unit = \xe1 (caseless) +Subject length lower bound = 2 + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + +/a|\x{c1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +Starting code units: A a \xc1 \xe1 +Subject length lower bound = 1 + \x{e1}xxx + 0: \xe1 + +/\x{c1}|\x{e1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Subject length lower bound = 1 + +/X(\x{e1})Y/ucp,replace=>\U$1<,substitute_extended + X\x{e1}Y + 1: >\xc1< + +/X(\x{121})Y/ucp,replace=>\U$1<,substitute_extended + X\x{121}Y + 1: >\x{120}< + +/s/i,ucp + \x{17f} + 0: \x{17f} + +/s/i,utf + \x{17f} + 0: \x{17f} + +/[^s]/i,ucp +\= Expect no match + \x{17f} +No match + +/[^s]/i,utf +\= Expect no match + \x{17f} +No match + +# ---------------------------------------------------- + # End of testinput12 diff --git a/dist2/testdata/testoutput12-32 b/dist2/testdata/testoutput12-32 index cb901907..03b6e394 100644 --- a/dist2/testdata/testoutput12-32 +++ b/dist2/testdata/testoutput12-32 @@ -1500,4 +1500,283 @@ Failed: error 142 at offset 4: syntax error in subpattern name (missing terminat /(?(á/utf Failed: error 142 at offset 4: syntax error in subpattern name (missing terminator?) +# Invalid UTF-16/32 tests. + +/.../g,match_invalid_utf + abcd\x{df00}wxzy\x{df00}pqrs + 0: abc + 0: wxz + 0: pqr + abcd\x{80}wxzy\x{df00}pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/abc/match_invalid_utf + ab\x{df00}ab\=ph +Partial match: ab +\= Expect no match + ab\x{df00}cdef\=ph +No match + +/ab$/match_invalid_utf + ab\x{df00}cdeab + 0: ab +\= Expect no match + ab\x{df00}cde +No match + +/.../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + 0: abc + 0: d\x{80}w + 0: xzy + 0: pqr + +/(?<=x)../g,match_invalid_utf + abcd\x{80}wxzy\x{df00}pqrs + 0: zy + abcd\x{80}wxzy\x{df00}xpqrs + 0: zy + 0: pq + +/X$/match_invalid_utf +\= Expect no match + X\x{df00} +No match + +/(?<=..)X/match_invalid_utf,aftertext + AB\x{df00}AQXYZ + 0: X + 0+ YZ + AB\x{df00}AQXYZ\=offset=5 + 0: X + 0+ YZ + AB\x{df00}\x{df00}AXYZXC\=offset=5 + 0: X + 0+ C +\= Expect no match + AB\x{df00}XYZ +No match + AB\x{df00}XYZ\=offset=3 +No match + AB\x{df00}AXYZ +No match + AB\x{df00}AXYZ\=offset=4 +No match + AB\x{df00}\x{df00}AXYZ\=offset=5 +No match + +/.../match_invalid_utf +\= Expect no match + A\x{d800}B +No match + A\x{110000}B +No match + +/aa/utf,ucp,match_invalid_utf,global + aa\x{d800}aa + 0: aa + 0: aa + +/aa/utf,ucp,match_invalid_utf,global + \x{d800}aa + 0: aa + +# ---------------------------------------------------- + +/(*UTF)(?=\x{123})/I +Capture group count = 0 +May match empty string +Compile options: <none> +Overall options: utf +First code unit = \x{123} +Subject length lower bound = 1 + +/[\x{c1}\x{e1}]X[\x{145}\x{146}]/I,utf +Capture group count = 0 +Options: utf +First code unit = \xc1 (caseless) +Last code unit = \x{145} (caseless) +Subject length lower bound = 3 + +/[\xff\x{ffff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +/[\xff\x{ff}]/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +/[\xff\x{ff}]/I +Capture group count = 0 +Starting code units: \xff +Subject length lower bound = 1 + +/[Ss]/I +Capture group count = 0 +First code unit = 'S' (caseless) +Subject length lower bound = 1 + +/[Ss]/I,utf +Capture group count = 0 +Options: utf +Starting code units: S s +Subject length lower bound = 1 + +/(?:\x{ff}|\x{3000})/I,utf +Capture group count = 0 +Options: utf +Starting code units: \xff +Subject length lower bound = 1 + +# ---------------------------------------------------- +# UCP and casing tests + +/\x{120}/i,I +Capture group count = 0 +Options: caseless +First code unit = \x{120} +Subject length lower bound = 1 + +/\x{c1}/i,I,ucp +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Subject length lower bound = 1 + +/[\x{120}\x{121}]/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{120} + Ket + End +------------------------------------------------------------------ + +/[ab\x{120}]+/iB,ucp +------------------------------------------------------------------ + Bra + [ABab\x{120}-\x{121}]++ + Ket + End +------------------------------------------------------------------ + aABb\x{121}\x{120} + 0: aABb\x{121}\x{120} + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} +No match + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + 0: \x{121}\xe1 + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + 0: \x{121}\xe1 + +/[^\x{120}]/i,no_start_optimize + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} +No match + +/[^\x{120}]/i + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} +No match + +/\x{120}{2}/i,ucp + \x{121}\x{121} + 0: \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} +No match + +/\x{c1}+\x{e1}/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + +/\x{c1}+\x{e1}/iIB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Last code unit = \xe1 (caseless) +Subject length lower bound = 2 + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + +/a|\x{c1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +Starting code units: A a \xc1 \xe1 +Subject length lower bound = 1 + \x{e1}xxx + 0: \xe1 + +/\x{c1}|\x{e1}/iI,ucp +Capture group count = 0 +Options: caseless ucp +First code unit = \xc1 (caseless) +Subject length lower bound = 1 + +/X(\x{e1})Y/ucp,replace=>\U$1<,substitute_extended + X\x{e1}Y + 1: >\xc1< + +/X(\x{121})Y/ucp,replace=>\U$1<,substitute_extended + X\x{121}Y + 1: >\x{120}< + +/s/i,ucp + \x{17f} + 0: \x{17f} + +/s/i,utf + \x{17f} + 0: \x{17f} + +/[^s]/i,ucp +\= Expect no match + \x{17f} +No match + +/[^s]/i,utf +\= Expect no match + \x{17f} +No match + +# ---------------------------------------------------- + # End of testinput12 diff --git a/dist2/testdata/testoutput14-16 b/dist2/testdata/testoutput14-16 index 05b7d48a..61541f61 100644 --- a/dist2/testdata/testoutput14-16 +++ b/dist2/testdata/testoutput14-16 @@ -1,9 +1,12 @@ -# These test special (mostly error) UTF features of DFA matching. They are a -# selection of the more comprehensive tests that are run for non-DFA matching. -# The output is different for the different widths. +# These test special UTF and UCP features of DFA matching. The output is +# different for the different widths. #subject dfa +# ---------------------------------------------------- +# These are a selection of the more comprehensive tests that are run for +# non-DFA matching. + /X/utf XX\x{d800} Failed: error -24: UTF-16 error: missing low surrogate at end at offset 2 @@ -30,7 +33,7 @@ Failed: error -26: UTF-16 error: isolated low surrogate at offset 2 XX\x{110000} ** Failed: character \x{110000} is greater than 0x10ffff and so cannot be converted to UTF-16 XX\x{d800}\x{1234} -Failed: error -25: UTF-16 error: invalid low surrogate at offset 3 +Failed: error -25: UTF-16 error: invalid low surrogate at offset 2 /badutf/utf X\xdf @@ -57,5 +60,66 @@ No match No match \xf7\x80\=ph No match + +# ---------------------------------------------------- +# UCP and casing tests - except for the first two, these will all fail in 8-bit +# mode because they are testing UCP without UTF and use characters > 255. + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} +No match + +/\x{c1}+\x{e1}/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + 1: \xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + 1: \xe1\xe1 + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + 0: \x{121}\xe1 + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + 0: \x{121}\xe1 + +/[^\x{120}]/i,no_start_optimize + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} +No match + +/[^\x{120}]/i + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} +No match + +/\x{120}{2}/i,ucp + \x{121}\x{121} + 0: \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} +No match + +# ---------------------------------------------------- # End of testinput14 diff --git a/dist2/testdata/testoutput14-32 b/dist2/testdata/testoutput14-32 index 30d7fa64..f1f65b74 100644 --- a/dist2/testdata/testoutput14-32 +++ b/dist2/testdata/testoutput14-32 @@ -1,9 +1,12 @@ -# These test special (mostly error) UTF features of DFA matching. They are a -# selection of the more comprehensive tests that are run for non-DFA matching. -# The output is different for the different widths. +# These test special UTF and UCP features of DFA matching. The output is +# different for the different widths. #subject dfa +# ---------------------------------------------------- +# These are a selection of the more comprehensive tests that are run for +# non-DFA matching. + /X/utf XX\x{d800} Failed: error -27: UTF-32 error: code points 0xd800-0xdfff are not defined at offset 2 @@ -57,5 +60,66 @@ No match No match \xf7\x80\=ph No match + +# ---------------------------------------------------- +# UCP and casing tests - except for the first two, these will all fail in 8-bit +# mode because they are testing UCP without UTF and use characters > 255. + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} +No match + +/\x{c1}+\x{e1}/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + 1: \xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + 1: \xe1\xe1 + +/\x{120}\x{c1}/i,ucp,no_start_optimize + \x{121}\x{e1} + 0: \x{121}\xe1 + +/\x{120}\x{c1}/i,ucp + \x{121}\x{e1} + 0: \x{121}\xe1 + +/[^\x{120}]/i,no_start_optimize + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +\= Expect no match + \x{121} +No match + +/[^\x{120}]/i + \x{121} + 0: \x{121} + +/[^\x{120}]/i,ucp +\= Expect no match + \x{121} +No match + +/\x{120}{2}/i,ucp + \x{121}\x{121} + 0: \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +\= Expect no match + \x{121}\x{121} +No match + +# ---------------------------------------------------- # End of testinput14 diff --git a/dist2/testdata/testoutput14-8 b/dist2/testdata/testoutput14-8 index 1fb0dc14..aa624141 100644 --- a/dist2/testdata/testoutput14-8 +++ b/dist2/testdata/testoutput14-8 @@ -1,9 +1,12 @@ -# These test special (mostly error) UTF features of DFA matching. They are a -# selection of the more comprehensive tests that are run for non-DFA matching. -# The output is different for the different widths. +# These test special UTF and UCP features of DFA matching. The output is +# different for the different widths. #subject dfa +# ---------------------------------------------------- +# These are a selection of the more comprehensive tests that are run for +# non-DFA matching. + /X/utf XX\x{d800} Failed: error -16: UTF-8 error: code points 0xd800-0xdfff are not defined at offset 2 @@ -57,5 +60,66 @@ Failed: error -3: UTF-8 error: 1 byte missing at end at offset 2 Failed: error -5: UTF-8 error: 3 bytes missing at end at offset 0 \xf7\x80\=ph Failed: error -4: UTF-8 error: 2 bytes missing at end at offset 0 + +# ---------------------------------------------------- +# UCP and casing tests - except for the first two, these will all fail in 8-bit +# mode because they are testing UCP without UTF and use characters > 255. + +/\x{c1}/i,no_start_optimize +\= Expect no match + \x{e1} +No match + +/\x{c1}+\x{e1}/iB,ucp +------------------------------------------------------------------ + Bra + /i \x{c1}+ + /i \x{e1} + Ket + End +------------------------------------------------------------------ + \x{c1}\x{c1}\x{c1} + 0: \xc1\xc1\xc1 + 1: \xc1\xc1 + \x{e1}\x{e1}\x{e1} + 0: \xe1\xe1\xe1 + 1: \xe1\xe1 + +/\x{120}\x{c1}/i,ucp,no_start_optimize +Failed: error 134 at offset 6: character code point value in \x{} or \o{} is too large + \x{121}\x{e1} + +/\x{120}\x{c1}/i,ucp +Failed: error 134 at offset 6: character code point value in \x{} or \o{} is too large + \x{121}\x{e1} + +/[^\x{120}]/i,no_start_optimize +Failed: error 134 at offset 8: character code point value in \x{} or \o{} is too large + \x{121} + +/[^\x{120}]/i,ucp,no_start_optimize +Failed: error 134 at offset 8: character code point value in \x{} or \o{} is too large +\= Expect no match + \x{121} + +/[^\x{120}]/i +Failed: error 134 at offset 8: character code point value in \x{} or \o{} is too large + \x{121} + +/[^\x{120}]/i,ucp +Failed: error 134 at offset 8: character code point value in \x{} or \o{} is too large +\= Expect no match + \x{121} + +/\x{120}{2}/i,ucp +Failed: error 134 at offset 6: character code point value in \x{} or \o{} is too large + \x{121}\x{121} + +/[^\x{120}]{2}/i,ucp +Failed: error 134 at offset 8: character code point value in \x{} or \o{} is too large +\= Expect no match + \x{121}\x{121} + +# ---------------------------------------------------- # End of testinput14 diff --git a/dist2/testdata/testoutput15 b/dist2/testdata/testoutput15 index d312765b..9154e5f9 100644 --- a/dist2/testdata/testoutput15 +++ b/dist2/testdata/testoutput15 @@ -45,7 +45,7 @@ Minimum heap limit = 0 Minimum match limit = 12 Minimum depth limit = 3 0: aaaaaaaaa - + /(a)(?1)++/ aa\=find_limits Minimum heap limit = 0 @@ -66,7 +66,7 @@ Minimum heap limit = 0 Minimum match limit = 24 Minimum depth limit = 3 0: abbbbbbbbbbbbbbbbbbbbba - + /a(?:.(*THEN))*?a/ims abbbbbbbbbbbbbbbbbbbbba\=find_limits Minimum heap limit = 0 @@ -207,9 +207,9 @@ No match \= Expect limit exceeded aaaaaaaaaaaaaz\=depth_limit=10 Failed: error -53: matching depth limit exceeded - -# These three have infinitely nested recursions. - + +# These three have infinitely nested recursions. + /((?2))((?1))/ abc Failed: error -52: nested recursion at the same subject position @@ -221,7 +221,7 @@ Failed: error -52: nested recursion at the same subject position /(?(R)a*(?1)|((?R))b)/ aaaabcde Failed: error -52: nested recursion at the same subject position - + # The allusedtext modifier does not work with JIT, which does not maintain # the leftchar/rightchar data. @@ -233,7 +233,7 @@ Failed: error -52: nested recursion at the same subject position 0: abcxyz >>> 0+ xyzpqr - + /(?<=pqr)abc(?=xyz)/allusedtext xyzpqrabcxyzpqr 0: pqrabcxyz @@ -242,12 +242,12 @@ Failed: error -52: nested recursion at the same subject position 0: pqrabcxyz <<< >>> 0+ xyzpqr - + /a\b/ a.\=allusedtext 0: a. > - a\=allusedtext + a\=allusedtext 0: a /abc\Kxyz/ @@ -264,7 +264,115 @@ Failed: error -52: nested recursion at the same subject position abcabcdefg 0: abcabcde >>>>> - + +#subject allusedtext + +/(?<=abc)123/ + xyzabc123pqr + 0: abc123 + <<< + xyzabc12\=ps +Partial match: abc12 + <<< + xyzabc12\=ph +Partial match: abc12 + <<< + +/\babc\b/ + +++abc+++ + 0: +abc+ + < > + +++ab\=ps +Partial match: +ab + < + +++ab\=ph +Partial match: +ab + < + +/(?<=abc)def/ + abc\=ph +Partial match: abc + <<< + +/(?<=123)(*MARK:xx)abc/mark + xxxx123a\=ph +Partial match, mark=xx: 123a + <<< + xxxx123a\=ps +Partial match, mark=xx: 123a + <<< + +/(?<=(?<=a)b)c.*/I +Capture group count = 0 +Max lookbehind = 1 +First code unit = 'c' +Subject length lower bound = 1 + abc\=ph +Partial match: abc + << +\= Expect no match + xbc\=ph +No match + +/(?<=ab)c.*/I +Capture group count = 0 +Max lookbehind = 2 +First code unit = 'c' +Subject length lower bound = 1 + abc\=ph +Partial match: abc + << +\= Expect no match + xbc\=ph +No match + +/abc(?<=bc)def/ + xxxabcd\=ph +Partial match: abcd + +/(?<=ab)cdef/ + xxabcd\=ph +Partial match: abcd + << + +/(?<=(?<=(?<=a)b)c)./I +Capture group count = 0 +Max lookbehind = 1 +Subject length lower bound = 1 + 123abcXYZ + 0: abcX + <<< + +/(?<=ab(cd(?<=...)))./I +Capture group count = 1 +Max lookbehind = 4 +Subject length lower bound = 1 + abcdX + 0: abcdX + <<<< + 1: cd + +/(?<=ab((?<=...)cd))./I +Capture group count = 1 +Max lookbehind = 4 +Subject length lower bound = 1 + ZabcdX + 0: ZabcdX + <<<<< + 1: cd + +/(?<=((?<=(?<=ab).))(?1)(?1))./I +Capture group count = 1 +Max lookbehind = 2 +Subject length lower bound = 1 + abxZ + 0: abxZ + <<< + 1: + +#subject +# ------------------------------------------------------------------- + # These tests provoke recursion loops, which give a different error message # when JIT is used. @@ -282,7 +390,7 @@ Subject length lower bound = 0 abcd 0: a 1: a - defg + defg Failed: error -52: nested recursion at the same subject position /(ab|(bc|(de|(?R))))/I @@ -292,7 +400,7 @@ Subject length lower bound = 0 abcd 0: ab 1: ab - fghi + fghi Failed: error -52: nested recursion at the same subject position /(ab|(bc|(de|(?1))))/I @@ -302,7 +410,7 @@ Subject length lower bound = 0 abcd 0: ab 1: ab - fghi + fghi Failed: error -52: nested recursion at the same subject position /x(ab|(bc|(de|(?1)x)x)x)/I @@ -312,17 +420,17 @@ Subject length lower bound = 3 xab123 0: xab 1: ab - xfghi + xfghi Failed: error -52: nested recursion at the same subject position /(?!\w)(?R)/ abcd Failed: error -52: nested recursion at the same subject position - =abc + =abc Failed: error -52: nested recursion at the same subject position /(?=\w)(?R)/ - =abc + =abc Failed: error -52: nested recursion at the same subject position abcd Failed: error -52: nested recursion at the same subject position @@ -339,7 +447,7 @@ Failed: error -52: nested recursion at the same subject position aaa 0: aaa 1: aaa - bbb + bbb Failed: error -52: nested recursion at the same subject position /[^\xff]((?1))/BI @@ -356,8 +464,8 @@ Capture group count = 1 Subject length lower bound = 1 abcd Failed: error -52: nested recursion at the same subject position - -# These tests don't behave the same with JIT + +# These tests don't behave the same with JIT /\w+(?C1)/BI,no_auto_possess ------------------------------------------------------------------ @@ -406,7 +514,7 @@ Subject length lower bound = 1 1 ^^ End of pattern No match -# This test breaks the JIT stack limit +# This test breaks the JIT stack limit /(|]+){2,2452}/ (|]+){2,2452} @@ -417,4 +525,12 @@ No match \[a]{60} Failed: error -63: heap limit exceeded +/b(?<!ax)(?!cx)/allusedtext + abc + 0: abc + < > + abcz + 0: abcz + < >> + # End of testinput15 diff --git a/dist2/testdata/testoutput2 b/dist2/testdata/testoutput2 index 2f91c389..dc170113 100644 --- a/dist2/testdata/testoutput2 +++ b/dist2/testdata/testoutput2 @@ -809,24 +809,7 @@ Subject length lower bound = 3 fooabar No match -# This one is here because Perl behaves differently; see also the following. - -/^(a\1?){4}$/I -Capture group count = 1 -Max back reference = 1 -Compile options: <none> -Overall options: anchored -Starting code units: a -Subject length lower bound = 4 -\= Expect no match - aaaa -No match - aaaaaa -No match - -# Perl does not fail these two for the final subjects. Neither did PCRE until -# release 8.01. The problem is in backtracking into a subpattern that contains -# a recursive reference to itself. PCRE has now made these into atomic patterns. +# Perl does not fail these two for the final subjects. /^(xa|=?\1a){2}$/ xa=xaa @@ -1480,7 +1463,7 @@ Subject length lower bound = 2 Capture group count = 0 First code unit = 'a' Last code unit = 'a' -Subject length lower bound = 1 +Subject length lower bound = 2 /(?=a)a.*/I Capture group count = 0 @@ -2252,7 +2235,7 @@ Failed: error 115 at offset 5: reference to non-existent subpattern Capture group count = 271 Max back reference = 270 Starting code units: 0 1 2 3 4 5 6 7 8 9 -Subject length lower bound = 0 +Subject length lower bound = 1 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 ABC ABC\=ovector=300 0: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 ABC ABC 1: 1 @@ -3406,7 +3389,7 @@ Subject length lower bound = 2 Capture group count = 0 May match empty string First code unit = 'a' -Subject length lower bound = 0 +Subject length lower bound = 1 /(?=abc).xyz/Ii Capture group count = 0 @@ -3425,7 +3408,7 @@ Subject length lower bound = 4 Capture group count = 0 May match empty string First code unit = 'a' -Subject length lower bound = 0 +Subject length lower bound = 1 /(?=.)a/I Capture group count = 0 @@ -3436,7 +3419,7 @@ Subject length lower bound = 1 Capture group count = 1 First code unit = 'a' Last code unit = 'a' -Subject length lower bound = 1 +Subject length lower bound = 2 /((?=abcda)ab)/I Capture group count = 1 @@ -6492,6 +6475,8 @@ Subject length lower bound = 2 0: aaaA5 aaaa5 0: aaaa5 + a5 + 0: a5 /(a*|b*)[cd]/I Capture group count = 1 @@ -9369,21 +9354,17 @@ Partial match: abc12 xyzabc123pqr 0: 123 xyzabc12\=ps -Partial match: abc12 - <<< +Partial match: 12 xyzabc12\=ph -Partial match: abc12 - <<< +Partial match: 12 /\babc\b/ +++abc+++ 0: abc +++ab\=ps -Partial match: +ab - < +Partial match: ab +++ab\=ph -Partial match: +ab - < +Partial match: ab /(?&word)(?&element)(?(DEFINE)(?<element><[^m][^>]>[^<])(?<word>\w*+))/B ------------------------------------------------------------------ @@ -10062,7 +10043,6 @@ No match ------------------------------------------------------------------ Bra ^ - Once CBra 1 ab CBra 2 @@ -10073,8 +10053,6 @@ No match Alt x Ket - Ket - Once CBra 1 ab CBra 2 @@ -10085,7 +10063,6 @@ No match Alt x Ket - Ket $ Ket End @@ -10401,8 +10378,7 @@ No match /(?<=abc)def/ abc\=ph -Partial match: abc - <<< +Partial match: /abc$/ abc @@ -10482,27 +10458,23 @@ Failed: error 168 at offset 3: \c must be followed by a printable ASCII characte /(?P<abn>(?P=abn)xxx)/B ------------------------------------------------------------------ Bra - Once CBra 1 \1 xxx Ket Ket - Ket End ------------------------------------------------------------------ /(a\1z)/B ------------------------------------------------------------------ Bra - Once CBra 1 a \1 z Ket Ket - Ket End ------------------------------------------------------------------ @@ -10780,7 +10752,7 @@ Capture group count = 0 Options: caseless First code unit = 'a' (caseless) Last code unit = 'a' (caseless) -Subject length lower bound = 1 +Subject length lower bound = 2 /(abc)\1+/ @@ -10990,6 +10962,12 @@ Matched, but too many substrings Assert abc Ket + Assert + abc + Ket + Assert + abc + Ket abc Ket End @@ -11001,6 +10979,10 @@ Matched, but too many substrings Assert abc Ket + Brazero + Assert + abc + Ket abc Ket End @@ -11009,9 +10991,15 @@ Matched, but too many substrings /(?=abc)++abc/B ------------------------------------------------------------------ Bra + Once + Assert + abc + Ket + Brazero Assert abc Ket + Ket abc Ket End @@ -11122,7 +11110,7 @@ Matched, but too many substrings ------------------------------------------------------------------ Bra Brazero - AssertB + Assert back Reverse CBra 1 abc @@ -11229,7 +11217,7 @@ Subject length lower bound = 0 /a(*SKIP)c|b(*ACCEPT)cd(*ACCEPT)|x/I Capture group count = 0 Starting code units: a b x -Subject length lower bound = 0 +Subject length lower bound = 1 ax 0: x @@ -11246,6 +11234,7 @@ No match /(*ACCEPT)a/I,aftertext Capture group count = 0 +May match empty string Subject length lower bound = 0 bax 0: @@ -11254,7 +11243,7 @@ Subject length lower bound = 0 /z(*ACCEPT)a/I,aftertext Capture group count = 0 First code unit = 'z' -Subject length lower bound = 0 +Subject length lower bound = 1 baxzbx 0: z 0+ bx @@ -11301,27 +11290,23 @@ No match /(?P<abn>(?P=abn)xxx)/B ------------------------------------------------------------------ Bra - Once CBra 1 \1 xxx Ket Ket - Ket End ------------------------------------------------------------------ /(a\1z)/B ------------------------------------------------------------------ Bra - Once CBra 1 a \1 z Ket Ket - Ket End ------------------------------------------------------------------ @@ -11959,11 +11944,9 @@ Callout 2: last capture = 0 /(?<=123)(*MARK:xx)abc/mark xxxx123a\=ph -Partial match, mark=xx: 123a - <<< +Partial match, mark=xx: a xxxx123a\=ps -Partial match, mark=xx: 123a - <<< +Partial match, mark=xx: a /123\Kabc/startchar xxxx123a\=ph @@ -13323,7 +13306,6 @@ Failed: error 144 at offset 5: subpattern name must start with a non-digit Bra Brazero SCBra 1 - Once CBra 2 CBra 3 a @@ -13335,7 +13317,6 @@ Failed: error 144 at offset 5: subpattern name must start with a non-digit Ket Recurse Ket - Ket KetRmax a?+ Ket @@ -13353,7 +13334,7 @@ Failed: error 144 at offset 5: subpattern name must start with a non-digit Ket red \b - AssertB + Assert back Reverse \w Ket @@ -13410,7 +13391,7 @@ Failed: error 133 at offset 7: parentheses are too deeply nested (stack check) Once \s*+ Ket - AssertB + Assert back Reverse \w Ket @@ -13572,7 +13553,6 @@ Subject length lower bound = 4 /abcd/I,no_start_optimize Capture group count = 0 Options: no_start_optimize -Subject length lower bound = 0 /(|ab)*?d/I Capture group count = 1 @@ -13588,7 +13568,6 @@ Subject length lower bound = 1 /(|ab)*?d/I,no_start_optimize Capture group count = 1 Options: no_start_optimize -Subject length lower bound = 0 abd 0: abd 1: ab @@ -14005,7 +13984,6 @@ Matched, but too many substrings /((?+1)(\1))/B ------------------------------------------------------------------ Bra - Once CBra 1 Recurse CBra 2 @@ -14013,7 +13991,6 @@ Matched, but too many substrings Ket Ket Ket - Ket End ------------------------------------------------------------------ @@ -14431,7 +14408,6 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Bra Any - Once CBra 1 Recurse Recurse @@ -14440,7 +14416,6 @@ Subject length lower bound = 1 Alt $ Ket - Ket CBra 2 Ket Ket @@ -14451,7 +14426,6 @@ Subject length lower bound = 1 ------------------------------------------------------------------ Bra Any - Once CBra 1 Recurse Recurse @@ -14463,7 +14437,6 @@ Subject length lower bound = 1 Alt $ Ket - Ket CBra 3 Ket Ket @@ -14609,7 +14582,7 @@ Subject length lower bound = 65535 Capture group count = 1 Max back reference = 1 Starting code units: a b -Subject length lower bound = 0 +Subject length lower bound = 1 /(?|(aaa)|(b))(?1)/I Capture group count = 1 @@ -14627,7 +14600,7 @@ Max back reference = 1 Named capture groups: a 1 Starting code units: a b -Subject length lower bound = 0 +Subject length lower bound = 1 /(?|(?'a'aaa)|(?'a'b))(?'a'cccc)\k'a'/I,dupnames Capture group count = 2 @@ -14638,7 +14611,7 @@ Named capture groups: Options: dupnames Starting code units: a b Last code unit = 'c' -Subject length lower bound = 0 +Subject length lower bound = 5 /ab{3cd/ ab{3cd @@ -14811,7 +14784,7 @@ No match /abcd/null_context abcd\=null_context 0: abcd -\= Expect error +\= Expect error - not allowed together abcd\=null_context,find_limits ** Not allowed together: find_limits null_context abcd\=allusedtext,startchar @@ -14821,6 +14794,14 @@ No match abcd 1: w\x0dx\x82y\xdbz(12\$34$$\x345$) +/abcd/replace=w\rx\x82y\o{333}z(\Q12\$34$$\x34\E5$$),substitute_extended,substitute_literal + >>abcd<< + 1: >>w\rx\x82y\o{333}z(\Q12\$34$$\x34\E5$$)<< + +/abcd/g,replace=\$1$2\,substitute_literal + XabcdYabcdZ + 2: X\$1$2\Y\$1$2\Z + /a(bc)(DE)/replace=a\u$1\U$1\E$1\l$2\L$2\Eab\Uab\LYZ\EDone,substitute_extended abcDE 1: aBcBCbcdEdeabAByzDone @@ -14838,6 +14819,8 @@ Failed: error -57 at offset 4 in replacement: bad escape sequence in replacement 1: b:b ac\=replace=${1:+$1\:$1:$2} 1: c + >>ac<<\=replace=${1:+$1\:$1:$2},substitute_literal + 1: >>${1:+$1\:$1:$2}<< /a(?:(b)|(c))/substitute_extended,replace=X${1:-1:-1}X${2:-2:-2} ab @@ -14892,6 +14875,16 @@ Failed: error -55 at offset 3 in replacement: requested value is not set /(aa)(BB)/substitute_extended,replace=\U$1\L$2\E$1..\U$1\l$2$1 aaBB 1: AAbbaa..AAbBaa + +/abcd/replace=wxyz,substitute_matched + abcd + 1: wxyz + pqrs + 0: pqrs + +/abcd/g + >abcd1234abcd5678<\=replace=wxyz,substitute_matched + 2: >wxyz1234wxyz5678< /^(o(\1{72}{\"{\\{00000059079}\d*){74}}){19}/I Capture group count = 2 @@ -16436,7 +16429,7 @@ Capture group count = 1 Max back reference = 1 First code unit = 'a' Last code unit = 'b' -Subject length lower bound = 1 +Subject length lower bound = 2 ab 0: ab 1: a @@ -16628,11 +16621,24 @@ No match /(?<=(?=.){4,5}x)/B ------------------------------------------------------------------ Bra - AssertB + Assert back Reverse Assert Any Ket + Assert + Any + Ket + Assert + Any + Ket + Assert + Any + Ket + Brazero + Assert + Any + Ket x Ket Ket @@ -16920,6 +16926,7 @@ Subject length lower bound = 3 /(*ACCEPT:XX)^abc/I Capture group count = 0 +May match empty string Subject length lower bound = 0 /abc/replace=xyz @@ -16934,6 +16941,709 @@ Subject length lower bound = 0 abc\=null_context ** Replacement callouts are not supported with null_context. +/\[()]{65535}()/expand +Failed: error 197 at offset 131071: too many capturing groups (maximum 65535) + +/\[()]{65535}(?<A>)/expand +Failed: error 197 at offset 131075: too many capturing groups (maximum 65535) + +/a(?:(*ACCEPT))??bc/ + abc + 0: abc + axy + 0: a + +/a(*ACCEPT)??bc/ + abc + 0: abc + axy + 0: a + +/a(*ACCEPT:XX)??bc/mark + abc + 0: abc + axy + 0: a +MK: XX + +/(*:\)?/ +Failed: error 109 at offset 5: quantifier does not follow a repeatable item + +/(*:\Q \E){5}/alt_verbnames +Failed: error 109 at offset 11: quantifier does not follow a repeatable item + +/(?=abc)/I +Capture group count = 0 +May match empty string +First code unit = 'a' +Last code unit = 'c' +Subject length lower bound = 2 + +/(?|(X)|(XY))\1abc/I +Capture group count = 1 +Max back reference = 1 +First code unit = 'X' +Last code unit = 'c' +Subject length lower bound = 4 + +/(?|(a)|(bcde))(c)\2/I +Capture group count = 2 +Max back reference = 2 +Starting code units: a b +Last code unit = 'c' +Subject length lower bound = 3 + +/(?|(a)|(bcde))(c)\1/I +Capture group count = 2 +Max back reference = 1 +Starting code units: a b +Last code unit = 'c' +Subject length lower bound = 2 + +/(?|(?'A'a)|(?'A'bcde))(?'B'c)\k'B'(?'A')/I,dupnames +Capture group count = 3 +Max back reference = 2 +Named capture groups: + A 1 + A 3 + B 2 +Options: dupnames +Starting code units: a b +Last code unit = 'c' +Subject length lower bound = 3 + +/(?|(?'A'a)|(?'A'bcde))(?'B'c)\k'A'(?'A')/I,dupnames +Capture group count = 3 +Max back reference = 3 +Named capture groups: + A 1 + A 3 + B 2 +Options: dupnames +Starting code units: a b +Last code unit = 'c' +Subject length lower bound = 2 + +/((a|)+)+Z/I +Capture group count = 2 +Starting code units: Z a +Last code unit = 'Z' +Subject length lower bound = 1 + +/((?=a))[abcd]/I +Capture group count = 1 +First code unit = 'a' +Subject length lower bound = 1 + +/A(?:(*ACCEPT))?B/info +Capture group count = 0 +First code unit = 'A' +Subject length lower bound = 1 + +/(A(*ACCEPT)??B)C/ + ABC + 0: ABC + 1: AB + AXY + 0: A + 1: A + +/(?<=(?<=a)b)c.*/I +Capture group count = 0 +Max lookbehind = 1 +First code unit = 'c' +Subject length lower bound = 1 + abc\=ph +Partial match: c +\= Expect no match + xbc\=ph +No match + +/(?<=ab)c.*/I +Capture group count = 0 +Max lookbehind = 2 +First code unit = 'c' +Subject length lower bound = 1 + abc\=ph +Partial match: c +\= Expect no match + xbc\=ph +No match + +/(?<=a(?<=a|a)c)/I +Capture group count = 0 +Max lookbehind = 2 +May match empty string +Subject length lower bound = 0 + +/(?<=a(?<=a|ba)c)/I +Capture group count = 0 +Max lookbehind = 2 +May match empty string +Subject length lower bound = 0 + +/(?<=(?<=a)b)(?<!abcd)/I +Capture group count = 0 +Max lookbehind = 4 +May match empty string +Subject length lower bound = 0 + +/(?<=(?<=a)b)(?<!abcd)(?<=(?<=a)bcde)/I +Capture group count = 0 +Max lookbehind = 4 +May match empty string +Subject length lower bound = 0 + +# Addition overflow +/( {32742} {42})(?<!\1{65481})/ +Failed: error 187 at offset 15: lookbehind assertion is too long + +# Multiplication overflow +/(X{65535})(?<=\1{32770})/ +Failed: error 187 at offset 10: lookbehind assertion is too long + +# ---- Non-atomic assertion tests ---- + +# Expect error: not allowed as a condition +/(?(*napla:xx)bc)/ +Failed: error 198 at offset 9: atomic assertion expected after (?( or (?(?C) + +/\A(*pla:.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 +No match + +/\A(*napla:.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 + 0: word1 word3 word1 word2 word3 word2 word2 word1 word3 + 1: word3 + +/\A(?*.*\b(\w++))(?>.*?\b\1\b){3}/ + word1 word3 word1 word2 word3 word2 word2 word1 word3 word4 + 0: word1 word3 word1 word2 word3 word2 word2 word1 word3 + 1: word3 + +/(*plb:(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + 0: b + 1: b + 2: <unset> + 3: b + abcda\=offset=4 +No match + +/(*naplb:(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + 0: b + 1: b + 2: <unset> + 3: b + abcda\=offset=4 + 0: a + 1: <unset> + 2: a + 3: a + +/(?<*(.)..|(.)...)(\1|\2)/ + abcdb\=offset=4 + 0: b + 1: b + 2: <unset> + 3: b + abcda\=offset=4 + 0: a + 1: <unset> + 2: a + 3: a + +/(*non_atomic_positive_lookahead:ab)/B +------------------------------------------------------------------ + Bra + Non-atomic assert + ab + Ket + Ket + End +------------------------------------------------------------------ + +/(*non_atomic_positive_lookbehind:ab)/B +------------------------------------------------------------------ + Bra + Non-atomic assert back + Reverse + ab + Ket + Ket + End +------------------------------------------------------------------ + +/(*pla:ab+)/B +------------------------------------------------------------------ + Bra + Assert + a + b++ + Ket + Ket + End +------------------------------------------------------------------ + +/(*napla:ab+)/B +------------------------------------------------------------------ + Bra + Non-atomic assert + a + b+ + Ket + Ket + End +------------------------------------------------------------------ + +/(*napla:)+/ + +/(*naplb:)+/ + +/(*napla:^x|^y)/I +Capture group count = 0 +May match empty string +Compile options: <none> +Overall options: anchored +Starting code units: x y +Subject length lower bound = 1 + +/(*napla:abc|abd)/I +Capture group count = 0 +May match empty string +First code unit = 'a' +Subject length lower bound = 1 + +/(*napla:a|(.)(*ACCEPT)zz)\1../ + abcd + 0: abc + 1: a + +/(*napla:a(*ACCEPT)zz|(.))\1../ + abcd + 0: bcd + 1: b + +/(*napla:a|(*COMMIT)(.))\1\1/ + aabc + 0: aa + 1: a +\= Expect no match + abbc +No match + +/(*napla:a|(.))\1\1/ + aabc + 0: aa + 1: a + abbc + 0: bb + 1: b + +# ---- + +# Expect error (recursion => not fixed length) +/(\2)((?=(?<=\1)))/ +Failed: error 125 at offset 8: lookbehind assertion is not fixed length + +/c*+(?<=[bc])/ + abc\=ph +Partial match: c + ab\=ph +Partial match: + abc\=ps + 0: c + ab\=ps + 0: + +/c++(?<=[bc])/ + abc\=ph +Partial match: c + ab\=ph +Partial match: + +/(?<=(?=.(?<=x)))/ + abx + 0: + ab\=ph +Partial match: + bxyz + 0: + xyz + 0: + +/\z/ + abc\=ph +Partial match: + abc\=ps + 0: + +/\Z/ + abc\=ph +Partial match: + abc\=ps + 0: + abc\n\=ph +Partial match: \x0a + abc\n\=ps + 0: + +/(?![ab]).*/ + ab\=ph +Partial match: + +/c*+/ + ab\=ph,offset=2 +Partial match: + +/\A\s*(a|(?:[^`]{28500}){4})/I +Capture group count = 1 +Max lookbehind = 1 +Compile options: <none> +Overall options: anchored +Subject length lower bound = 1 + a + 0: a + 1: a + +/\A\s*((?:[^`]{28500}){4})/I +Capture group count = 1 +Max lookbehind = 1 +Compile options: <none> +Overall options: anchored +Subject length lower bound = 65535 + +/\A\s*((?:[^`]{28500}){4}|a)/I +Capture group count = 1 +Max lookbehind = 1 +Compile options: <none> +Overall options: anchored +Subject length lower bound = 1 + a + 0: a + 1: a + +/(?<A>a)(?(<A>)b)((?<=b).*)/B +------------------------------------------------------------------ + Bra + CBra 1 + a + Ket + Cond + 1 Cond ref + b + Ket + CBra 2 + Assert back + Reverse + b + Ket + Any*+ + Ket + Ket + End +------------------------------------------------------------------ + +/(?(1)b)((?<=b).*)/B +------------------------------------------------------------------ + Bra + Cond + 1 Cond ref + b + Ket + CBra 1 + Assert back + Reverse + b + Ket + Any*+ + Ket + Ket + End +------------------------------------------------------------------ + +/(?(R1)b)((?<=b).*)/B +------------------------------------------------------------------ + Bra + Cond + Cond recurse 1 + b + Ket + CBra 1 + Assert back + Reverse + b + Ket + Any*+ + Ket + Ket + End +------------------------------------------------------------------ + +/(?(DEFINE)b)((?<=b).*)/B +------------------------------------------------------------------ + Bra + Cond + Cond false + b + Ket + CBra 1 + Assert back + Reverse + b + Ket + Any*+ + Ket + Ket + End +------------------------------------------------------------------ + +/(?(VERSION=10.4)b)((?<=b).*)/B +------------------------------------------------------------------ + Bra + Cond + Cond false + b + Ket + CBra 1 + Assert back + Reverse + b + Ket + Any*+ + Ket + Ket + End +------------------------------------------------------------------ + +/[aA]b[cC]/IB +------------------------------------------------------------------ + Bra + /i a + b + /i c + Ket + End +------------------------------------------------------------------ +Capture group count = 0 +First code unit = 'a' (caseless) +Last code unit = 'c' (caseless) +Subject length lower bound = 3 + +/[cc]abcd/I +Capture group count = 0 +First code unit = 'c' +Last code unit = 'd' +Subject length lower bound = 5 + +/[Cc]abcd/I +Capture group count = 0 +First code unit = 'C' (caseless) +Last code unit = 'd' +Subject length lower bound = 5 + +/[c]abcd/I +Capture group count = 0 +First code unit = 'c' +Last code unit = 'd' +Subject length lower bound = 5 + +/(?:c|C)abcd/I +Capture group count = 0 +First code unit = 'C' (caseless) +Last code unit = 'd' +Subject length lower bound = 5 + +/(a)?a/I +Capture group count = 1 +Starting code units: a +Last code unit = 'a' +Subject length lower bound = 1 + manm + 0: a + +/^(?|(\*)(*napla:\S*_(\2?+.+))|(\w)(?=\S*_(\2?+\1)))+_\2$/ + *abc_12345abc + 0: *abc_12345abc + 1: c + 2: 12345abc + +/^(?|(\*)(*napla:\S*_(\3?+.+))|(\w)(?=\S*_((\2?+\1))))+_\2$/ + *abc_12345abc + 0: *abc_12345abc + 1: c + 2: 12345abc + 3: 12345abc + +/^((\1+)(?C)|\d)+133X$/ + 111133X\=callout_capture +Callout 0: last capture = 2 + 1: 1 + 2: 111 +--->111133X + ^ ^ | +Callout 0: last capture = 2 + 1: 3 + 2: 3 +--->111133X + ^ ^ | +Callout 0: last capture = 2 + 1: 1 + 2: 11 +--->111133X + ^ ^ | +Callout 0: last capture = 2 + 1: 3 + 2: 3 +--->111133X + ^ ^ | + 0: 111133X + 1: 11 + 2: 11 + +/abc/replace=xyz,substitute_replacement_only + 123abc456 + 1: xyz + +/a(?<ONE>b)c(?<TWO>d)e/g,replace=X$ONE+${TWO}Z,substitute_replacement_only + "abcde-abcde-" + 2: Xb+dZXb+dZ + +/a(b)c|xyz/g,replace=<$0>,substitute_callout,substitute_replacement_only + abcdefabcpqr + 1(2) Old 0 3 "abc" New 0 5 "<abc>" + 2(2) Old 6 9 "abc" New 5 10 "<abc>" + 2: <abc><abc> + abxyzpqrabcxyz + 1(1) Old 2 5 "xyz" New 0 5 "<xyz>" + 2(2) Old 8 11 "abc" New 5 10 "<abc>" + 3(1) Old 11 14 "xyz" New 10 15 "<xyz>" + 3: <xyz><abc><xyz> + 12abc34xyz99abc55\=substitute_stop=2 + 1(2) Old 2 5 "abc" New 0 5 "<abc>" + 2(1) Old 7 10 "xyz" New 5 10 "<xyz> STOPPED" + 2: <abc> + 12abc34xyz99abc55\=substitute_skip=1 + 1(2) Old 2 5 "abc" New 0 5 "<abc> SKIPPED" + 2(1) Old 7 10 "xyz" New 0 5 "<xyz>" + 3(2) Old 12 15 "abc" New 5 10 "<abc>" + 3: <xyz><abc> + 12abc34xyz99abc55\=substitute_skip=2 + 1(2) Old 2 5 "abc" New 0 5 "<abc>" + 2(1) Old 7 10 "xyz" New 5 10 "<xyz> SKIPPED" + 3(2) Old 12 15 "abc" New 5 10 "<abc>" + 3: <abc><abc> + +/a(..)d/replace=>$1<,substitute_matched + xyzabcdxyzabcdxyz + 1: xyz>bc<xyzabcdxyz + xyzabcdxyzabcdxyz\=ovector=2 + 1: xyz>bc<xyzabcdxyz +\= Expect error + xyzabcdxyzabcdxyz\=ovector=1 +Failed: error -54 at offset 3 in replacement: requested value is not available + +/a(..)d/g,replace=>$1<,substitute_matched + xyzabcdxyzabcdxyz + 2: xyz>bc<xyz>bc<xyz + xyzabcdxyzabcdxyz\=ovector=2 + 2: xyz>bc<xyz>bc<xyz +\= Expect error + xyzabcdxyzabcdxyz\=ovector=1 +Failed: error -54 at offset 3 in replacement: requested value is not available + xyzabcdxyzabcdxyz\=ovector=1,substitute_unset_empty +Failed: error -54 at offset 3 in replacement: requested value is not available + +/55|a(..)d/g,replace=>$1<,substitute_matched + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + 3: xyz><>bc<xyz>bc<xyz +\= Expect error + xyz55abcdxyzabcdxyz\=ovector=2 +Failed: error -55 at offset 3 in replacement: requested value is not set + +/55|a(..)d/replace=>$1<,substitute_matched + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + 1: xyz><abcdxyzabcdxyz + +/55|a(..)d/replace=>$1< + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + 1: xyz><abcdxyzabcdxyz + +/55|a(..)d/g,replace=>$1< + xyz55abcdxyzabcdxyz\=ovector=2,substitute_unset_empty + 3: xyz><>bc<xyz>bc<xyz + +# Expect non-fixed-length error + +"(?<=X(?(DEFINE)(.*))(?1))." +Failed: error 125 at offset 0: lookbehind assertion is not fixed length + +/\sxxx\s/tables=1 +\= Expect no match + AB\x{85}xxx\x{a0}XYZ +No match + +/\sxxx\s/tables=2 + AB\x{85}xxx\x{a0}XYZ + 0: \x85xxx\xa0 + +/^\w+/tables=2 + École + 0: \xc3 + +/^\w+/tables=3 +** 'Tables = 3' is invalid: binary tables have not been loaded + École + +#loadtables ./testbtables + +/^\w+/tables=3 + École + 0: \xc3 + +/"(*MARK:>" 00 "<).."/hex,mark,no_start_optimize + AB + 0: AB +MK: >\x00< + A\=ph +Partial match, mark=>\x00<: A +\= Expect no match + A +No match, mark = >\x00< + +/"(*MARK:>" 00 "<).(?C1)."/hex,mark,no_start_optimize + AB +--->AB + 1 ^^ . +Latest Mark: >\x00< + 0: AB +MK: >\x00< + +/(?(VERSION=0.0/ +Failed: error 179 at offset 14: syntax error or number too big in (?(VERSION condition + +# Perl has made \K in lookarounds an error. At the moment PCRE2 still accepts. + +/(?=a\Kb)ab/ + ab + 0: b + +/(?!a\Kb)ac/ + ac + 0: ac + +/^abc(?<=b\Kc)d/ + abcd + 0: cd + +/^abc(?<!b\Kq)d/ + abcd + 0: abcd + +# --------- + # End of testinput2 Error -70: PCRE2_ERROR_BADDATA (unknown error number) Error -62: bad serialized data diff --git a/dist2/testdata/testoutput22-16 b/dist2/testdata/testoutput22-16 index 4b6008ce..54218540 100644 --- a/dist2/testdata/testoutput22-16 +++ b/dist2/testdata/testoutput22-16 @@ -9,7 +9,7 @@ Contains \C Options: utf First code unit = 'a' Last code unit = 'e' -Subject length lower bound = 0 +Subject length lower bound = 2 abXde 0: abXde diff --git a/dist2/testdata/testoutput22-8 b/dist2/testdata/testoutput22-8 index 5dd167ec..eab410eb 100644 --- a/dist2/testdata/testoutput22-8 +++ b/dist2/testdata/testoutput22-8 @@ -9,7 +9,7 @@ Contains \C Options: utf First code unit = 'a' Last code unit = 'e' -Subject length lower bound = 0 +Subject length lower bound = 2 abXde 0: abXde diff --git a/dist2/testdata/testoutput4 b/dist2/testdata/testoutput4 index 84b8b9ee..f43d9405 100644 --- a/dist2/testdata/testoutput4 +++ b/dist2/testdata/testoutput4 @@ -3,6 +3,10 @@ # some of the property tests may differ because of different versions of # Unicode in use by PCRE2 and Perl. +# WARNING: Use only / as the pattern delimiter. Although pcre2test supports +# a number of delimiters, all those other than / give problems with the +# perltest.sh script. + #newline_default lf anycrlf any #perltest @@ -795,7 +799,7 @@ No match ሴ 0: \x{1234} -"(?s)(.{1,5})"utf +/(?s)(.{1,5})/utf abcdefg 0: abcde 1: abcde @@ -1342,13 +1346,13 @@ No match 0: \x{4db4} \x{4db5} 0: \x{4db5} + \x{4db6} + 0: \x{4db6} \= Expect no match a No match \x{2b0} No match - \x{4db6} -No match /^\p{Lt}/utf \x{1c5} @@ -3596,7 +3600,7 @@ No match abcdefghijklmno 0: a -"[\S\V\H]"utf +/[\S\V\H]/utf /[^\p{Any}]*+x/utf x @@ -4012,4 +4016,20 @@ No match 0: 123123 1: 123 +/\X*/ + \xF3aaa\xE4\xEA\xEB\xFEa + 0: \xf3aaa\xe4\xea\xeb\xfea + +/Я/i,utf + \x{42f} + 0: \x{42f} + \x{44f} + 0: \x{44f} + +/(?=Я)/i,utf + \x{42f} + 0: + \x{44f} + 0: + # End of testinput4 diff --git a/dist2/testdata/testoutput5 b/dist2/testdata/testoutput5 index 5d64d00c..c2f8c3dd 100644 --- a/dist2/testdata/testoutput5 +++ b/dist2/testdata/testoutput5 @@ -474,7 +474,6 @@ Subject length lower bound = 0 Capture group count = 0 Compile options: no_start_optimize utf Overall options: anchored no_start_optimize utf -Subject length lower bound = 0 /()()()()()()()()()() ()()()()()()()()()() @@ -4018,7 +4017,7 @@ MK: a\x{12345}b\x{09}(d)c ------------------------------------------------------------------ Bra \b - AssertB + Assert back Reverse prop Xwd Ket @@ -4197,7 +4196,7 @@ Failed: error 125 at offset 2: lookbehind assertion is not fixed length ------------------------------------------------------------------ Bra ^ - AssertB not + Assert back not Assert \x{10385c} Ket @@ -4737,8 +4736,6 @@ No match No match /^\p{Common}/utf - \x{589} - 0: \x{589} \x{60c} 0: \x{60c} \x{61f} @@ -4829,7 +4826,7 @@ MK: ABC /(?<!)(*sr:)/B ------------------------------------------------------------------ Bra - AssertB not + Assert back not Ket Script run Ket @@ -4840,7 +4837,7 @@ MK: ABC /(?<=abc(?=X(*sr:BXY)CCC)XBXYCCC)./B ------------------------------------------------------------------ Bra - AssertB + Assert back Reverse abc Assert @@ -4894,6 +4891,18 @@ MK: ABC 0: \x{1d7ce}\x{1d7cf},, A\x{1d7ce}BC Latin fancy-common-digit Latin Latin 0: A\x{1d7ce}BC + +# Some Unicode 12.1.0 new script characters + +/\p{Elymaic}\p{Nandinagari}\p{Nyiakeng_Puachue_Hmong}\p{Wancho}/utf + \x{10fe5}\x{119AC}\x{1E10E}\x{1E2D1} + 0: \x{10fe5}\x{119ac}\x{1e10e}\x{1e2d1} + +# Some Unicode 13.0.0 new script characters + +/\p{Chorasmian}\p{Dives_Akuru}\p{Khitan_Small_Script}\p{Yezidi}/utf + \x{10FB0}\x{11900}\x{18B00}\x{10E80} + 0: \x{10fb0}\x{11900}\x{18b00}\x{10e80} # ------- @@ -4933,4 +4942,11 @@ Overall options: anchored Last code unit = 'z' Subject length lower bound = 3 +/(|ß)7/caseless,ucp + +/(\xc1)\1/i,ucp + \xc1\xe1\=no_jit + 0: \xc1\xe1 + 1: \xc1 + # End of testinput5 diff --git a/dist2/testdata/testoutput6 b/dist2/testdata/testoutput6 index 3ef10b40..607b572b 100644 --- a/dist2/testdata/testoutput6 +++ b/dist2/testdata/testoutput6 @@ -876,7 +876,7 @@ Partial match: abc 0: def /(?<=foo)bar/ - foob\=ps,offset=2 + foob\=ps,offset=2,allusedtext Partial match: foob <<< foobar...\=ps,dfa_restart,offset=4 @@ -6803,9 +6803,10 @@ Partial match: dogs xyzabc123pqr Failed: error -42: pattern contains an item that is not supported for DFA matching -/(?<=abc)123/ +/(?<=abc)123/allusedtext xyzabc123pqr - 0: 123 + 0: abc123 + <<< xyzabc12\=ps Partial match: abc12 <<< @@ -6813,9 +6814,10 @@ Partial match: abc12 Partial match: abc12 <<< -/\babc\b/ +/\babc\b/allusedtext +++abc+++ - 0: abc + 0: +abc+ + < > +++ab\=ps Partial match: +ab < @@ -6845,7 +6847,6 @@ No match /(abc|def|xyz)/I,no_start_optimize Capture group count = 1 Options: no_start_optimize -Subject length lower bound = 0 terhjk;abcdaadsfe 0: abc the quick xyz brown fox @@ -6933,7 +6934,7 @@ Failed: error -42: pattern contains an item that is not supported for DFA matchi ac Failed: error -42: pattern contains an item that is not supported for DFA matching -/(?<=abc)def/ +/(?<=abc)def/allusedtext abc\=ph Partial match: abc <<< @@ -7808,4 +7809,90 @@ No match 0 No match +/(?<=pqr)abc(?=xyz)/ + 123pqrabcxy\=ps,allusedtext +Partial match: pqrabcxy + <<< + 123pqrabcxyz\=ps,allusedtext + 0: pqrabcxyz + <<< >>> + +/(?>a+b)/ + aaaa\=ps +Partial match: aaaa + aaaab\=ps + 0: aaaab + +/(abc)(?1)/ + abca\=ps +Partial match: abca + abcabc\=ps + 0: abcabc + +/(?(?=abc).*|Z)/ + ab\=ps +Partial match: ab + abcxyz\=ps + 0: abcxyz + +/(abc)++x/ + abcab\=ps +Partial match: abcab + abc\=ps +Partial match: abc + ab\=ps +Partial match: ab + abcx + 0: abcx + +/\z/ + abc\=ph +Partial match: + abc\=ps + 0: + +/\Z/ + abc\=ph +Partial match: + abc\=ps + 0: + abc\n\=ph +Partial match: \x0a + abc\n\=ps + 0: + +/c*+(?<=[bc])/ + abc\=ph +Partial match: c + ab\=ph +Partial match: + abc\=ps + 0: c + ab\=ps + 0: + +/c++(?<=[bc])/ + abc\=ph +Partial match: c + ab\=ph +Partial match: + +/(?<=(?=.(?<=x)))/ + abx + 0: + ab\=ph +Partial match: + bxyz + 0: + xyz + 0: + +/(?![ab]).*/ + ab\=ph +Partial match: + +/c*+/ + ab\=ph,offset=2 +Partial match: + # End of testinput6 diff --git a/dist2/testdata/testoutput8-16-2 b/dist2/testdata/testoutput8-16-2 index 5706e60f..569a8603 100644 --- a/dist2/testdata/testoutput8-16-2 +++ b/dist2/testdata/testoutput8-16-2 @@ -720,41 +720,37 @@ Memory allocation (code space): 14 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 39 Bra + 0 35 Bra 2 Brazero - 3 32 SCBra 1 - 6 27 Once - 8 12 CBra 2 - 11 7 CBra 3 - 14 a - 16 \2 - 18 7 Ket - 20 11 Alt - 22 5 CBra 4 - 25 a* - 27 5 Ket - 29 22 Recurse - 31 23 Ket - 33 27 Ket - 35 32 KetRmax - 37 a?+ - 39 39 Ket - 41 End + 3 28 SCBra 1 + 6 12 CBra 2 + 9 7 CBra 3 + 12 a + 14 \2 + 16 7 Ket + 18 11 Alt + 20 5 CBra 4 + 23 a* + 25 5 Ket + 27 20 Recurse + 29 23 Ket + 31 28 KetRmax + 33 a?+ + 35 35 Ket + 37 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 20 Bra - 2 16 Once - 4 12 CBra 1 - 7 9 Recurse - 9 5 CBra 2 - 12 \1 - 14 5 Ket - 16 12 Ket - 18 16 Ket - 20 20 Ket - 22 End + 0 16 Bra + 2 12 CBra 1 + 5 7 Recurse + 7 5 CBra 2 + 10 \1 + 12 5 Ket + 14 12 Ket + 16 16 Ket + 18 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 14 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 28 Bra + 0 24 Bra 2 Any - 3 18 Once - 5 7 CBra 1 - 8 23 Recurse - 10 0 Recurse - 12 4 Alt - 14 \1 - 16 3 Alt - 18 $ - 19 14 Ket - 21 18 Ket - 23 3 CBra 2 - 26 3 Ket - 28 28 Ket - 30 End + 3 7 CBra 1 + 6 19 Recurse + 8 0 Recurse + 10 4 Alt + 12 \1 + 14 3 Alt + 16 $ + 17 14 Ket + 19 3 CBra 2 + 22 3 Ket + 24 24 Ket + 26 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 35 Bra + 0 31 Bra 2 Any - 3 25 Once - 5 14 CBra 1 - 8 30 Recurse - 10 0 Recurse - 12 3 CBra 2 - 15 3 Ket - 17 12 Recurse - 19 4 Alt - 21 \1 - 23 3 Alt - 25 $ - 26 21 Ket - 28 25 Ket - 30 3 CBra 3 - 33 3 Ket - 35 35 Ket - 37 End + 3 14 CBra 1 + 6 26 Recurse + 8 0 Recurse + 10 3 CBra 2 + 13 3 Ket + 15 10 Recurse + 17 4 Alt + 19 \1 + 21 3 Alt + 23 $ + 24 21 Ket + 26 3 CBra 3 + 29 3 Ket + 31 31 Ket + 33 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1020,4 +1012,9 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand +Failed: error 120 at offset 131070: regular expression is too large + # End of testinput8 diff --git a/dist2/testdata/testoutput8-16-3 b/dist2/testdata/testoutput8-16-3 index bd80a23f..80ee1c99 100644 --- a/dist2/testdata/testoutput8-16-3 +++ b/dist2/testdata/testoutput8-16-3 @@ -720,41 +720,37 @@ Memory allocation (code space): 18 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 52 Bra + 0 46 Bra 3 Brazero - 4 43 SCBra 1 - 8 36 Once - 11 15 CBra 2 - 15 8 CBra 3 - 19 a - 21 \2 - 23 8 Ket - 26 15 Alt - 29 6 CBra 4 - 33 a* - 35 6 Ket - 38 29 Recurse - 41 30 Ket - 44 36 Ket - 47 43 KetRmax - 50 a?+ - 52 52 Ket - 55 End + 4 37 SCBra 1 + 8 15 CBra 2 + 12 8 CBra 3 + 16 a + 18 \2 + 20 8 Ket + 23 15 Alt + 26 6 CBra 4 + 30 a* + 32 6 Ket + 35 26 Recurse + 38 30 Ket + 41 37 KetRmax + 44 a?+ + 46 46 Ket + 49 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 28 Bra - 3 22 Once - 6 16 CBra 1 - 10 13 Recurse - 13 6 CBra 2 - 17 \1 - 19 6 Ket - 22 16 Ket - 25 22 Ket - 28 28 Ket - 31 End + 0 22 Bra + 3 16 CBra 1 + 7 10 Recurse + 10 6 CBra 2 + 14 \1 + 16 6 Ket + 19 16 Ket + 22 22 Ket + 25 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 18 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 39 Bra + 0 33 Bra 3 Any - 4 25 Once - 7 10 CBra 1 - 11 32 Recurse - 14 0 Recurse - 17 5 Alt - 20 \1 - 22 4 Alt - 25 $ - 26 19 Ket - 29 25 Ket - 32 4 CBra 2 - 36 4 Ket - 39 39 Ket - 42 End + 4 10 CBra 1 + 8 26 Recurse + 11 0 Recurse + 14 5 Alt + 17 \1 + 19 4 Alt + 22 $ + 23 19 Ket + 26 4 CBra 2 + 30 4 Ket + 33 33 Ket + 36 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 49 Bra + 0 43 Bra 3 Any - 4 35 Once - 7 20 CBra 1 - 11 42 Recurse - 14 0 Recurse - 17 4 CBra 2 - 21 4 Ket - 24 17 Recurse - 27 5 Alt - 30 \1 - 32 4 Alt - 35 $ - 36 29 Ket - 39 35 Ket - 42 4 CBra 3 - 46 4 Ket - 49 49 Ket - 52 End + 4 20 CBra 1 + 8 36 Recurse + 11 0 Recurse + 14 4 CBra 2 + 18 4 Ket + 21 14 Recurse + 24 5 Alt + 27 \1 + 29 4 Alt + 32 $ + 33 29 Ket + 36 4 CBra 3 + 40 4 Ket + 43 43 Ket + 46 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-16-4 b/dist2/testdata/testoutput8-16-4 index bd80a23f..80ee1c99 100644 --- a/dist2/testdata/testoutput8-16-4 +++ b/dist2/testdata/testoutput8-16-4 @@ -720,41 +720,37 @@ Memory allocation (code space): 18 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 52 Bra + 0 46 Bra 3 Brazero - 4 43 SCBra 1 - 8 36 Once - 11 15 CBra 2 - 15 8 CBra 3 - 19 a - 21 \2 - 23 8 Ket - 26 15 Alt - 29 6 CBra 4 - 33 a* - 35 6 Ket - 38 29 Recurse - 41 30 Ket - 44 36 Ket - 47 43 KetRmax - 50 a?+ - 52 52 Ket - 55 End + 4 37 SCBra 1 + 8 15 CBra 2 + 12 8 CBra 3 + 16 a + 18 \2 + 20 8 Ket + 23 15 Alt + 26 6 CBra 4 + 30 a* + 32 6 Ket + 35 26 Recurse + 38 30 Ket + 41 37 KetRmax + 44 a?+ + 46 46 Ket + 49 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 28 Bra - 3 22 Once - 6 16 CBra 1 - 10 13 Recurse - 13 6 CBra 2 - 17 \1 - 19 6 Ket - 22 16 Ket - 25 22 Ket - 28 28 Ket - 31 End + 0 22 Bra + 3 16 CBra 1 + 7 10 Recurse + 10 6 CBra 2 + 14 \1 + 16 6 Ket + 19 16 Ket + 22 22 Ket + 25 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 18 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 39 Bra + 0 33 Bra 3 Any - 4 25 Once - 7 10 CBra 1 - 11 32 Recurse - 14 0 Recurse - 17 5 Alt - 20 \1 - 22 4 Alt - 25 $ - 26 19 Ket - 29 25 Ket - 32 4 CBra 2 - 36 4 Ket - 39 39 Ket - 42 End + 4 10 CBra 1 + 8 26 Recurse + 11 0 Recurse + 14 5 Alt + 17 \1 + 19 4 Alt + 22 $ + 23 19 Ket + 26 4 CBra 2 + 30 4 Ket + 33 33 Ket + 36 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 49 Bra + 0 43 Bra 3 Any - 4 35 Once - 7 20 CBra 1 - 11 42 Recurse - 14 0 Recurse - 17 4 CBra 2 - 21 4 Ket - 24 17 Recurse - 27 5 Alt - 30 \1 - 32 4 Alt - 35 $ - 36 29 Ket - 39 35 Ket - 42 4 CBra 3 - 46 4 Ket - 49 49 Ket - 52 End + 4 20 CBra 1 + 8 36 Recurse + 11 0 Recurse + 14 4 CBra 2 + 18 4 Ket + 21 14 Recurse + 24 5 Alt + 27 \1 + 29 4 Alt + 32 $ + 33 29 Ket + 36 4 CBra 3 + 40 4 Ket + 43 43 Ket + 46 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-32-2 b/dist2/testdata/testoutput8-32-2 index a86ef9a5..91d96c94 100644 --- a/dist2/testdata/testoutput8-32-2 +++ b/dist2/testdata/testoutput8-32-2 @@ -720,41 +720,37 @@ Memory allocation (code space): 28 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 39 Bra + 0 35 Bra 2 Brazero - 3 32 SCBra 1 - 6 27 Once - 8 12 CBra 2 - 11 7 CBra 3 - 14 a - 16 \2 - 18 7 Ket - 20 11 Alt - 22 5 CBra 4 - 25 a* - 27 5 Ket - 29 22 Recurse - 31 23 Ket - 33 27 Ket - 35 32 KetRmax - 37 a?+ - 39 39 Ket - 41 End + 3 28 SCBra 1 + 6 12 CBra 2 + 9 7 CBra 3 + 12 a + 14 \2 + 16 7 Ket + 18 11 Alt + 20 5 CBra 4 + 23 a* + 25 5 Ket + 27 20 Recurse + 29 23 Ket + 31 28 KetRmax + 33 a?+ + 35 35 Ket + 37 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 20 Bra - 2 16 Once - 4 12 CBra 1 - 7 9 Recurse - 9 5 CBra 2 - 12 \1 - 14 5 Ket - 16 12 Ket - 18 16 Ket - 20 20 Ket - 22 End + 0 16 Bra + 2 12 CBra 1 + 5 7 Recurse + 7 5 CBra 2 + 10 \1 + 12 5 Ket + 14 12 Ket + 16 16 Ket + 18 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 28 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 28 Bra + 0 24 Bra 2 Any - 3 18 Once - 5 7 CBra 1 - 8 23 Recurse - 10 0 Recurse - 12 4 Alt - 14 \1 - 16 3 Alt - 18 $ - 19 14 Ket - 21 18 Ket - 23 3 CBra 2 - 26 3 Ket - 28 28 Ket - 30 End + 3 7 CBra 1 + 6 19 Recurse + 8 0 Recurse + 10 4 Alt + 12 \1 + 14 3 Alt + 16 $ + 17 14 Ket + 19 3 CBra 2 + 22 3 Ket + 24 24 Ket + 26 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 35 Bra + 0 31 Bra 2 Any - 3 25 Once - 5 14 CBra 1 - 8 30 Recurse - 10 0 Recurse - 12 3 CBra 2 - 15 3 Ket - 17 12 Recurse - 19 4 Alt - 21 \1 - 23 3 Alt - 25 $ - 26 21 Ket - 28 25 Ket - 30 3 CBra 3 - 33 3 Ket - 35 35 Ket - 37 End + 3 14 CBra 1 + 6 26 Recurse + 8 0 Recurse + 10 3 CBra 2 + 13 3 Ket + 15 10 Recurse + 17 4 Alt + 19 \1 + 21 3 Alt + 23 $ + 24 21 Ket + 26 3 CBra 3 + 29 3 Ket + 31 31 Ket + 33 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-32-3 b/dist2/testdata/testoutput8-32-3 index a86ef9a5..91d96c94 100644 --- a/dist2/testdata/testoutput8-32-3 +++ b/dist2/testdata/testoutput8-32-3 @@ -720,41 +720,37 @@ Memory allocation (code space): 28 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 39 Bra + 0 35 Bra 2 Brazero - 3 32 SCBra 1 - 6 27 Once - 8 12 CBra 2 - 11 7 CBra 3 - 14 a - 16 \2 - 18 7 Ket - 20 11 Alt - 22 5 CBra 4 - 25 a* - 27 5 Ket - 29 22 Recurse - 31 23 Ket - 33 27 Ket - 35 32 KetRmax - 37 a?+ - 39 39 Ket - 41 End + 3 28 SCBra 1 + 6 12 CBra 2 + 9 7 CBra 3 + 12 a + 14 \2 + 16 7 Ket + 18 11 Alt + 20 5 CBra 4 + 23 a* + 25 5 Ket + 27 20 Recurse + 29 23 Ket + 31 28 KetRmax + 33 a?+ + 35 35 Ket + 37 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 20 Bra - 2 16 Once - 4 12 CBra 1 - 7 9 Recurse - 9 5 CBra 2 - 12 \1 - 14 5 Ket - 16 12 Ket - 18 16 Ket - 20 20 Ket - 22 End + 0 16 Bra + 2 12 CBra 1 + 5 7 Recurse + 7 5 CBra 2 + 10 \1 + 12 5 Ket + 14 12 Ket + 16 16 Ket + 18 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 28 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 28 Bra + 0 24 Bra 2 Any - 3 18 Once - 5 7 CBra 1 - 8 23 Recurse - 10 0 Recurse - 12 4 Alt - 14 \1 - 16 3 Alt - 18 $ - 19 14 Ket - 21 18 Ket - 23 3 CBra 2 - 26 3 Ket - 28 28 Ket - 30 End + 3 7 CBra 1 + 6 19 Recurse + 8 0 Recurse + 10 4 Alt + 12 \1 + 14 3 Alt + 16 $ + 17 14 Ket + 19 3 CBra 2 + 22 3 Ket + 24 24 Ket + 26 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 35 Bra + 0 31 Bra 2 Any - 3 25 Once - 5 14 CBra 1 - 8 30 Recurse - 10 0 Recurse - 12 3 CBra 2 - 15 3 Ket - 17 12 Recurse - 19 4 Alt - 21 \1 - 23 3 Alt - 25 $ - 26 21 Ket - 28 25 Ket - 30 3 CBra 3 - 33 3 Ket - 35 35 Ket - 37 End + 3 14 CBra 1 + 6 26 Recurse + 8 0 Recurse + 10 3 CBra 2 + 13 3 Ket + 15 10 Recurse + 17 4 Alt + 19 \1 + 21 3 Alt + 23 $ + 24 21 Ket + 26 3 CBra 3 + 29 3 Ket + 31 31 Ket + 33 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-32-4 b/dist2/testdata/testoutput8-32-4 index a86ef9a5..91d96c94 100644 --- a/dist2/testdata/testoutput8-32-4 +++ b/dist2/testdata/testoutput8-32-4 @@ -720,41 +720,37 @@ Memory allocation (code space): 28 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 39 Bra + 0 35 Bra 2 Brazero - 3 32 SCBra 1 - 6 27 Once - 8 12 CBra 2 - 11 7 CBra 3 - 14 a - 16 \2 - 18 7 Ket - 20 11 Alt - 22 5 CBra 4 - 25 a* - 27 5 Ket - 29 22 Recurse - 31 23 Ket - 33 27 Ket - 35 32 KetRmax - 37 a?+ - 39 39 Ket - 41 End + 3 28 SCBra 1 + 6 12 CBra 2 + 9 7 CBra 3 + 12 a + 14 \2 + 16 7 Ket + 18 11 Alt + 20 5 CBra 4 + 23 a* + 25 5 Ket + 27 20 Recurse + 29 23 Ket + 31 28 KetRmax + 33 a?+ + 35 35 Ket + 37 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 20 Bra - 2 16 Once - 4 12 CBra 1 - 7 9 Recurse - 9 5 CBra 2 - 12 \1 - 14 5 Ket - 16 12 Ket - 18 16 Ket - 20 20 Ket - 22 End + 0 16 Bra + 2 12 CBra 1 + 5 7 Recurse + 7 5 CBra 2 + 10 \1 + 12 5 Ket + 14 12 Ket + 16 16 Ket + 18 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 28 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 28 Bra + 0 24 Bra 2 Any - 3 18 Once - 5 7 CBra 1 - 8 23 Recurse - 10 0 Recurse - 12 4 Alt - 14 \1 - 16 3 Alt - 18 $ - 19 14 Ket - 21 18 Ket - 23 3 CBra 2 - 26 3 Ket - 28 28 Ket - 30 End + 3 7 CBra 1 + 6 19 Recurse + 8 0 Recurse + 10 4 Alt + 12 \1 + 14 3 Alt + 16 $ + 17 14 Ket + 19 3 CBra 2 + 22 3 Ket + 24 24 Ket + 26 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 35 Bra + 0 31 Bra 2 Any - 3 25 Once - 5 14 CBra 1 - 8 30 Recurse - 10 0 Recurse - 12 3 CBra 2 - 15 3 Ket - 17 12 Recurse - 19 4 Alt - 21 \1 - 23 3 Alt - 25 $ - 26 21 Ket - 28 25 Ket - 30 3 CBra 3 - 33 3 Ket - 35 35 Ket - 37 End + 3 14 CBra 1 + 6 26 Recurse + 8 0 Recurse + 10 3 CBra 2 + 13 3 Ket + 15 10 Recurse + 17 4 Alt + 19 \1 + 21 3 Alt + 23 $ + 24 21 Ket + 26 3 CBra 3 + 29 3 Ket + 31 31 Ket + 33 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-8-2 b/dist2/testdata/testoutput8-8-2 index 33fd6e3e..8393d5c5 100644 --- a/dist2/testdata/testoutput8-8-2 +++ b/dist2/testdata/testoutput8-8-2 @@ -720,41 +720,37 @@ Memory allocation (code space): 10 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 57 Bra + 0 51 Bra 3 Brazero - 4 48 SCBra 1 - 9 40 Once - 12 18 CBra 2 - 17 10 CBra 3 - 22 a - 24 \2 - 27 10 Ket - 30 16 Alt - 33 7 CBra 4 - 38 a* - 40 7 Ket - 43 33 Recurse - 46 34 Ket - 49 40 Ket - 52 48 KetRmax - 55 a?+ - 57 57 Ket - 60 End + 4 42 SCBra 1 + 9 18 CBra 2 + 14 10 CBra 3 + 19 a + 21 \2 + 24 10 Ket + 27 16 Alt + 30 7 CBra 4 + 35 a* + 37 7 Ket + 40 30 Recurse + 43 34 Ket + 46 42 KetRmax + 49 a?+ + 51 51 Ket + 54 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 31 Bra - 3 25 Once - 6 19 CBra 1 - 11 14 Recurse - 14 8 CBra 2 - 19 \1 - 22 8 Ket - 25 19 Ket - 28 25 Ket - 31 31 Ket - 34 End + 0 25 Bra + 3 19 CBra 1 + 8 11 Recurse + 11 8 CBra 2 + 16 \1 + 19 8 Ket + 22 19 Ket + 25 25 Ket + 28 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 10 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 42 Bra + 0 36 Bra 3 Any - 4 27 Once - 7 11 CBra 1 - 12 34 Recurse - 15 0 Recurse - 18 6 Alt - 21 \1 - 24 4 Alt - 27 $ - 28 21 Ket - 31 27 Ket - 34 5 CBra 2 - 39 5 Ket - 42 42 Ket - 45 End + 4 11 CBra 1 + 9 28 Recurse + 12 0 Recurse + 15 6 Alt + 18 \1 + 21 4 Alt + 24 $ + 25 21 Ket + 28 5 CBra 2 + 33 5 Ket + 36 36 Ket + 39 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 53 Bra + 0 47 Bra 3 Any - 4 38 Once - 7 22 CBra 1 - 12 45 Recurse - 15 0 Recurse - 18 5 CBra 2 - 23 5 Ket - 26 18 Recurse - 29 6 Alt - 32 \1 - 35 4 Alt - 38 $ - 39 32 Ket - 42 38 Ket - 45 5 CBra 3 - 50 5 Ket - 53 53 Ket - 56 End + 4 22 CBra 1 + 9 39 Recurse + 12 0 Recurse + 15 5 CBra 2 + 20 5 Ket + 23 15 Recurse + 26 6 Alt + 29 \1 + 32 4 Alt + 35 $ + 36 32 Ket + 39 5 CBra 3 + 44 5 Ket + 47 47 Ket + 50 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1020,4 +1012,9 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand +Failed: error 120 at offset 131070: regular expression is too large + # End of testinput8 diff --git a/dist2/testdata/testoutput8-8-3 b/dist2/testdata/testoutput8-8-3 index 59c69250..963700a3 100644 --- a/dist2/testdata/testoutput8-8-3 +++ b/dist2/testdata/testoutput8-8-3 @@ -720,41 +720,37 @@ Memory allocation (code space): 12 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 70 Bra + 0 62 Bra 4 Brazero - 5 59 SCBra 1 - 11 49 Once - 15 21 CBra 2 - 21 11 CBra 3 - 27 a - 29 \2 - 32 11 Ket - 36 20 Alt - 40 8 CBra 4 - 46 a* - 48 8 Ket - 52 40 Recurse - 56 41 Ket - 60 49 Ket - 64 59 KetRmax - 68 a?+ - 70 70 Ket - 74 End + 5 51 SCBra 1 + 11 21 CBra 2 + 17 11 CBra 3 + 23 a + 25 \2 + 28 11 Ket + 32 20 Alt + 36 8 CBra 4 + 42 a* + 44 8 Ket + 48 36 Recurse + 52 41 Ket + 56 51 KetRmax + 60 a?+ + 62 62 Ket + 66 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 39 Bra - 4 31 Once - 8 23 CBra 1 - 14 18 Recurse - 18 9 CBra 2 - 24 \1 - 27 9 Ket - 31 23 Ket - 35 31 Ket - 39 39 Ket - 43 End + 0 31 Bra + 4 23 CBra 1 + 10 14 Recurse + 14 9 CBra 2 + 20 \1 + 23 9 Ket + 27 23 Ket + 31 31 Ket + 35 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 12 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 53 Bra + 0 45 Bra 4 Any - 5 34 Once - 9 14 CBra 1 - 15 43 Recurse - 19 0 Recurse - 23 7 Alt - 27 \1 - 30 5 Alt - 34 $ - 35 26 Ket - 39 34 Ket - 43 6 CBra 2 - 49 6 Ket - 53 53 Ket - 57 End + 5 14 CBra 1 + 11 35 Recurse + 15 0 Recurse + 19 7 Alt + 23 \1 + 26 5 Alt + 30 $ + 31 26 Ket + 35 6 CBra 2 + 41 6 Ket + 45 45 Ket + 49 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 67 Bra + 0 59 Bra 4 Any - 5 48 Once - 9 28 CBra 1 - 15 57 Recurse - 19 0 Recurse - 23 6 CBra 2 - 29 6 Ket - 33 23 Recurse - 37 7 Alt - 41 \1 - 44 5 Alt - 48 $ - 49 40 Ket - 53 48 Ket - 57 6 CBra 3 - 63 6 Ket - 67 67 Ket - 71 End + 5 28 CBra 1 + 11 49 Recurse + 15 0 Recurse + 19 6 CBra 2 + 25 6 Ket + 29 19 Recurse + 33 7 Alt + 37 \1 + 40 5 Alt + 44 $ + 45 40 Ket + 49 6 CBra 3 + 55 6 Ket + 59 59 Ket + 63 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/testoutput8-8-4 b/dist2/testdata/testoutput8-8-4 index 94808656..8e19908e 100644 --- a/dist2/testdata/testoutput8-8-4 +++ b/dist2/testdata/testoutput8-8-4 @@ -720,41 +720,37 @@ Memory allocation (code space): 14 /(((a\2)|(a*)\g<-1>))*a?/ ------------------------------------------------------------------ - 0 83 Bra + 0 73 Bra 5 Brazero - 6 70 SCBra 1 - 13 58 Once - 18 24 CBra 2 - 25 12 CBra 3 - 32 a - 34 \2 - 37 12 Ket - 42 24 Alt - 47 9 CBra 4 - 54 a* - 56 9 Ket - 61 47 Recurse - 66 48 Ket - 71 58 Ket - 76 70 KetRmax - 81 a?+ - 83 83 Ket - 88 End + 6 60 SCBra 1 + 13 24 CBra 2 + 20 12 CBra 3 + 27 a + 29 \2 + 32 12 Ket + 37 24 Alt + 42 9 CBra 4 + 49 a* + 51 9 Ket + 56 42 Recurse + 61 48 Ket + 66 60 KetRmax + 71 a?+ + 73 73 Ket + 78 End ------------------------------------------------------------------ /((?+1)(\1))/ ------------------------------------------------------------------ - 0 47 Bra - 5 37 Once - 10 27 CBra 1 - 17 22 Recurse - 22 10 CBra 2 - 29 \1 - 32 10 Ket - 37 27 Ket - 42 37 Ket - 47 47 Ket - 52 End + 0 37 Bra + 5 27 CBra 1 + 12 17 Recurse + 17 10 CBra 2 + 24 \1 + 27 10 Ket + 32 27 Ket + 37 37 Ket + 42 End ------------------------------------------------------------------ "(?1)(?#?'){2}(a)" @@ -771,45 +767,41 @@ Memory allocation (code space): 14 /.((?2)(?R)|\1|$)()/ ------------------------------------------------------------------ - 0 64 Bra + 0 54 Bra 5 Any - 6 41 Once - 11 17 CBra 1 - 18 52 Recurse - 23 0 Recurse - 28 8 Alt - 33 \1 - 36 6 Alt - 41 $ - 42 31 Ket - 47 41 Ket - 52 7 CBra 2 - 59 7 Ket - 64 64 Ket - 69 End + 6 17 CBra 1 + 13 42 Recurse + 18 0 Recurse + 23 8 Alt + 28 \1 + 31 6 Alt + 36 $ + 37 31 Ket + 42 7 CBra 2 + 49 7 Ket + 54 54 Ket + 59 End ------------------------------------------------------------------ /.((?3)(?R)()(?2)|\1|$)()/ ------------------------------------------------------------------ - 0 81 Bra + 0 71 Bra 5 Any - 6 58 Once - 11 34 CBra 1 - 18 69 Recurse - 23 0 Recurse - 28 7 CBra 2 - 35 7 Ket - 40 28 Recurse - 45 8 Alt - 50 \1 - 53 6 Alt - 58 $ - 59 48 Ket - 64 58 Ket - 69 7 CBra 3 - 76 7 Ket - 81 81 Ket - 86 End + 6 34 CBra 1 + 13 59 Recurse + 18 0 Recurse + 23 7 CBra 2 + 30 7 Ket + 35 23 Recurse + 40 8 Alt + 45 \1 + 48 6 Alt + 53 $ + 54 48 Ket + 59 7 CBra 3 + 66 7 Ket + 71 71 Ket + 76 End ------------------------------------------------------------------ /(?1)()((((((\1++))\x85)+)|))/ @@ -1019,4 +1011,8 @@ Failed: error 114 at offset 509: missing closing parenthesis /([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00]([00](*ACCEPT)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))/-fullbincode +#pattern -fullbincode + +/\[()]{65535}/expand + # End of testinput8 diff --git a/dist2/testdata/wintestoutput3 b/dist2/testdata/wintestoutput3 index be856b16..b1894b66 100644 --- a/dist2/testdata/wintestoutput3 +++ b/dist2/testdata/wintestoutput3 @@ -88,13 +88,13 @@ No match 0: école /\w/I -Capturing subpattern count = 0 +Capture group count = 0 Starting code units: 0 1 2 3 4 5 6 7 8 9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z _ a b c d e f g h i j k l m n o p q r s t u v w x y z Subject length lower bound = 1 /\w/I,locale=french -Capturing subpattern count = 0 +Capture group count = 0 Starting code units: 0 1 2 3 4 5 6 7 8 9 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z _ a b c d e f g h i j k l m n o p q r s t u v w x y z ƒ Š Œ Ž š œ ž Ÿ ª ² ³ µ ¹ º À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö @@ -165,7 +165,7 @@ No match Ket End ------------------------------------------------------------------ -Capturing subpattern count = 0 +Capture group count = 0 Starting code units: A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a b c d e f g h i j k l m n o p q r s t u v w x y z ƒ Š Œ Ž š œ ž Ÿ ª µ º À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å |