
# This file was generated by the build system used internally in the Yandex monorepo.
# Only simple modifications are allowed (adding source-files to targets, adding simple properties
# like target_include_directories). These modifications will be ported to original
# ya.make files by maintainers. Any complex modifications which can't be ported back to the
# original buildsystem will not be accepted.


get_built_tool_path(
  TOOL_archiver_bin
  TOOL_archiver_dependency
  tools/archiver
  archiver
)
get_built_tool_path(
  TOOL_triecompiler_bin
  TOOL_triecompiler_dependency
  tools/triecompiler
  triecompiler
)

add_library(library-cpp-tokenizer)
target_compile_options(library-cpp-tokenizer PRIVATE
  -DCATBOOST_OPENSOURCE=yes
)
target_link_libraries(library-cpp-tokenizer PUBLIC
  contrib-libs-linux-headers
  contrib-libs-cxxsupp
  yutil
  library-cpp-enumbitset
  library-cpp-langmask
  cpp-token-lite
)
target_sources(library-cpp-tokenizer PRIVATE
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/abbreviations.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/charclasses.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/multitokenparser.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/multitokenutil.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/nlpparser.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/sentbreakfilter.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/split.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/tokenizer.cpp
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/special_tokens.cpp
)
add_custom_command(
  OUTPUT
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/SpecialTokens.rodata
  DEPENDS
  ${TOOL_archiver_bin}
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/special_tokens.trie
  COMMAND
  ${TOOL_archiver_bin}
  -q
  -p
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/special_tokens.trie:
  -o
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/SpecialTokens.rodata
)
add_custom_command(
  OUTPUT
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/special_tokens.trie
  DEPENDS
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/special_tokens.txt
  ${TOOL_triecompiler_bin}
  WORKING_DIRECTORY
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer
  COMMAND
  ${TOOL_triecompiler_bin}
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/special_tokens.trie
  -0
  -i
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/special_tokens.txt
  -w
)
target_ragel_lexers(library-cpp-tokenizer
  PRIVATE
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/nlptok_v2.rl6
  -L
  -G2
)
target_ragel_lexers(library-cpp-tokenizer
  PRIVATE
  ${CMAKE_SOURCE_DIR}/library/cpp/tokenizer/nlptok_v3.rl6
  -L
  -G2
)
target_rodata_sources(library-cpp-tokenizer
  PRIVATE
  ${CMAKE_BINARY_DIR}/library/cpp/tokenizer/SpecialTokens.rodata
)
if(NOT CMAKE_CROSSCOMPILING)
  add_dependencies(library-cpp-tokenizer
    archiver
    triecompiler
)
endif()
