diff --git a/.github/workflows/rapidyaml.yml b/.github/workflows/rapidyaml.yml new file mode 100644 index 000000000..b89fd865c --- /dev/null +++ b/.github/workflows/rapidyaml.yml @@ -0,0 +1,107 @@ +name: rapidyaml + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - main + pull_request: + +jobs: + + # check that the spec'ed version of rapidyaml passes its own tests + ryml: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: checkout rapidyaml + run: | + cd rapidyaml/native + make rapidyaml + - name: configure + run: | + cd rapidyaml/native/rapidyaml + cmake -B build -D CMAKE_BUILD_TYPE=${{matrix.bt}} -D RYML_BUILD_TESTS=ON + - name: build + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-build --parallel --verbose + - name: run tests + run: | + cd rapidyaml/native/rapidyaml + cmake --build build --target ryml-test-run + + # run the c++ tests, also in Debug to test with assertions + cpp: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: check jni header up to date + run: make -C rapidyaml/native -B jni jnicheck + - name: get rapidyaml + run: make -C rapidyaml/native rapidyaml + - name: run c++ tests, static ----------------------------- + run: echo + - name: cfg c++, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-static + - name: build c++ tests, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-static + - name: run c++ tests, static + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static + - name: run c++ tests, static with timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-static-timing + - name: run c++ tests, shared ----------------------------- + run: echo + - name: cfg c++, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native cfg-shared + - name: build c++ tests, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native build-shared + - name: run c++ tests, shared + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared + - name: run c++ tests, shared with timing + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml/native test-shared-timing + + # run the java tests, also in Debug to test with assertions + java: + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - bt: Debug + - bt: Release + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: build lib + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml build + - name: build jar + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml jar + - name: run java tests + run: RAPIDYAML_BUILD_TYPE=${{matrix.bt}} make -C rapidyaml test diff --git a/.github/workflows/ys.yml b/.github/workflows/ys.yml new file mode 100644 index 000000000..ff35b0bc2 --- /dev/null +++ b/.github/workflows/ys.yml @@ -0,0 +1,65 @@ +name: ys + +defaults: + run: + shell: bash -xeo pipefail {0} +'on': + workflow_dispatch: null + push: + branches: + - main + pull_request: + +jobs: + + # run ys tests, also in Debug to test with assertions. + # TODO: add a job to do all the downloads and build graalvm, and + # then make the other jobs depend on that + ys: + name: ys/${{matrix.ysparser}}/${{matrix.bt}} + runs-on: ubuntu-24.04 + if: always() + continue-on-error: false + strategy: + fail-fast: false + matrix: + include: + - {v: 1, ysparser: snake} + - {v: 1, ysparser: rapid-arr, bt: Debug} + - {v: 1, ysparser: rapid-arr, bt: Release} + - {v: 1, ysparser: rapid-buf, bt: Debug} + - {v: 1, ysparser: rapid-buf, bt: Release} + steps: + - name: checkout (action) + uses: actions/checkout@v4 + with: {submodules: recursive, fetch-depth: 0} # use fetch-depth to ensure all tags are fetched + - name: run core tests + run: | + . .profile + make test-core \ + v=${{matrix.v}} \ + YS_TESTING=1 \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run ys/test-run + run: | + . .profile + make -C ys test-run \ + v=${{matrix.v}} \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run test-ys + run: | + . .profile + make test-ys \ + v=${{matrix.v}} \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} + - name: run all tests + run: | + . .profile + make test \ + v=${{matrix.v}} \ + YS_TESTING=1 \ + YS_PARSER=${{matrix.ysparser}} \ + RAPIDYAML_BUILD_TYPE=${{matrix.bt}} diff --git a/.gitignore b/.gitignore index d5629f25d..cb8881acc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vimrc /*.clj /*.md *.md.tmp diff --git a/Makefile b/Makefile index 72cc80595..6af0b89bf 100644 --- a/Makefile +++ b/Makefile @@ -22,12 +22,14 @@ BINDINGS := \ rust \ DIRS := \ + rapidyaml \ core \ libyamlscript \ $(BINDINGS) \ ys \ BUILD_DIRS := \ + rapidyaml \ libyamlscript \ go \ nodejs \ @@ -49,9 +51,6 @@ PUBLISH := $(DIRS:%=publish-%) CLEAN := $(DIRS:%=clean-%) REALCLEAN := $(DIRS:%=realclean-%) DISTCLEAN := $(DIRS:%=distclean-%) -DOCKER_BUILD := $(DIRS:%=docker-build-%) -DOCKER_TEST := $(DIRS:%=docker-test-%) -DOCKER_SHELL := $(DIRS:%=docker-shell-%) export HEAD := $(shell git rev-parse HEAD) @@ -142,7 +141,7 @@ test: $(TEST) test-core: $(MAKE) -C core test v=$v test-ys: - $(MAKE) -C ys test-all v=$v GRAALVM_O=b + $(MAKE) -C ys test v=$v GRAALVM_O=b test-%: % $(MAKE) -C $< test v=$v GRAALVM_O=b test-unit: @@ -262,7 +261,9 @@ bump: $(BUILD_BIN_YS) $(CLEAN): clean: $(CLEAN) - $(RM) -r libyamlscript/lib ys/bin $(MAVEN_REPOSITORY)/yamlscript + $(RM) -r $(MAVEN_REPOSITORY)/yamlscript + $(RM) -r $(MAVEN_REPOSITORY)/org/yamlscript + $(RM) -r libyamlscript/lib ys/bin $(RM) -r libyamlscript-0* ys-0* yamlscript.cli-*.jar $(RM) -r sample/advent/hearsay-rust/target/ $(RM) -r homebrew-yamlscript @@ -289,24 +290,6 @@ distclean-%: % $(MAKE) -C $< distclean $(RM) -r .calva/ .clj-kondo/.cache .lsp/ -# XXX Limit removing ~/.m2 to ingy until we can get ~/.m2 to not be used sysclean: realclean $(RM) -r $(YS_TMP) $(RM) -r /tmp/yamlscript-* /tmp/ys-local -ifeq (ingy,$(USER)) - $(RM) -r $(HOME)/.m2 -endif - -$(DOCKER_BUILD): -docker-build: $(DOCKER_BUILD) -docker-build-%: % - $(MAKE) -C $< docker-build - -$(DOCKER_TEST): -docker-test: $(DOCKER_TEST) -docker-test-%: % - $(MAKE) -C $< docker-test v=$v - -$(DOCKER_SHELL): -docker-shell-%: % - $(MAKE) -C $< docker-shell v=$v diff --git a/clojure/Makefile b/clojure/Makefile index 70ace4810..3c6921e51 100644 --- a/clojure/Makefile +++ b/clojure/Makefile @@ -8,7 +8,7 @@ export CLOJARS_PASSWORD ?= $(shell util/get-setting password) #------------------------------------------------------------------------------ -test install deploy:: $(LEIN) $(YAMLSCRIPT_JAVA_INSTALLED) +test install deploy:: $(LEIN) $< $@ release: deploy diff --git a/common/base.mk b/common/base.mk index 06158290b..756004e0a 100644 --- a/common/base.mk +++ b/common/base.mk @@ -41,10 +41,6 @@ chown:: clean-all:: $(MAKE) -C $(ROOT) $@ -docker-build:: - -docker-test:: - always: env:: diff --git a/common/binding.mk b/common/binding.mk index bbd6b15ba..5a6c1a7ab 100644 --- a/common/binding.mk +++ b/common/binding.mk @@ -1,12 +1,11 @@ -include $(COMMON)/docker.mk +include $(COMMON)/vars-rapidyaml.mk +include $(COMMON)/vars-libys.mk -test:: $(LIBYAMLSCRIPT_SO_FQNP) +test:: $(LIBYS_SO_FQNP) -$(LIBYAMLSCRIPT_SO_FQNP): | $(ROOT)/libyamlscript +$(LIBYS_SO_FQNP): | $(ROOT)/libyamlscript $(MAKE) -C $(ROOT)/libyamlscript build -export PATH := $(BUILD_BIN):$(PATH) - build-doc:: build-bin build-bin: diff --git a/common/clojure.mk b/common/clojure.mk index 158fc0c37..d75792763 100644 --- a/common/clojure.mk +++ b/common/clojure.mk @@ -45,7 +45,6 @@ LEIN_REPL_OPTIONS := \ #------------------------------------------------------------------------------ clean:: - $(RM) Dockerfile $(RM) -r .lein-* $(RM) -r reports/ target/ @@ -128,7 +127,7 @@ ifdef PORT repl-port := :port $(PORT) endif -.nrepl-pid: $(LEIN) +.nrepl-pid: $(LEIN) repl-deps ( \ $< $(LEIN_REPL_OPTIONS) repl :headless $(repl-port) & \ echo $$! > $@ \ diff --git a/common/docker-apt-base.dockerfile b/common/docker-apt-base.dockerfile deleted file mode 100644 index a27315f9d..000000000 --- a/common/docker-apt-base.dockerfile +++ /dev/null @@ -1,22 +0,0 @@ - -RUN set -x \ - && apt-get update \ - && apt-get install -y \ - apt-file \ - apt-transport-https \ - autoconf \ - automake \ - bash \ - build-essential \ - cpio \ - curl \ - dialog \ - git \ - iputils-ping \ - libtool \ - pkgconf \ - unzip \ - wget \ - zip \ - zlib1g-dev \ - && true diff --git a/common/docker-apt-clojure.dockerfile b/common/docker-apt-clojure.dockerfile deleted file mode 100644 index 5e408a67d..000000000 --- a/common/docker-apt-clojure.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - leiningen \ - && true diff --git a/common/docker-apt-dev.dockerfile b/common/docker-apt-dev.dockerfile deleted file mode 100644 index e6e8e3d07..000000000 --- a/common/docker-apt-dev.dockerfile +++ /dev/null @@ -1,14 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - bash-completion \ - less \ - locales \ - silversearcher-ag \ - tig \ - tmate \ - tmux \ - vim \ - zsh \ - zsh-common \ - && true diff --git a/common/docker-apt-node.dockerfile b/common/docker-apt-node.dockerfile deleted file mode 100644 index ef832fd8b..000000000 --- a/common/docker-apt-node.dockerfile +++ /dev/null @@ -1,12 +0,0 @@ - -RUN set +x \ - && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh \ - | bash \ - && export NVM_DIR=$HOME/.nvm \ - && ls -la $HOME \ - && source "$NVM_DIR/nvm.sh" \ - && nvm install --lts \ - && npm install -g yarn \ - && echo "source $NVM_DIR/nvm.sh" >> $HOME/.bashrc \ - && echo "source $HOME/.bashrc" >> $HOME/.bash_profile \ - && true diff --git a/common/docker-apt-perl.dockerfile b/common/docker-apt-perl.dockerfile deleted file mode 100644 index 5212db2e1..000000000 --- a/common/docker-apt-perl.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - cpanminus \ - && true diff --git a/common/docker-apt-python.dockerfile b/common/docker-apt-python.dockerfile deleted file mode 100644 index 97d6623fb..000000000 --- a/common/docker-apt-python.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ - -RUN set -x \ - && sudo apt-get install -y \ - python3 \ - python3-pytest \ - python3-venv \ - && true diff --git a/common/docker-copy-project-clj.dockerfile b/common/docker-copy-project-clj.dockerfile deleted file mode 100644 index 422ecbef8..000000000 --- a/common/docker-copy-project-clj.dockerfile +++ /dev/null @@ -1,2 +0,0 @@ - -COPY project.clj /tmp/project.clj diff --git a/common/docker-copy-project-deps.dockerfile b/common/docker-copy-project-deps.dockerfile deleted file mode 100644 index f76e81f00..000000000 --- a/common/docker-copy-project-deps.dockerfile +++ /dev/null @@ -1,2 +0,0 @@ - -COPY .project.clj /tmp/project.clj diff --git a/common/docker-deps-clojure.dockerfile b/common/docker-deps-clojure.dockerfile deleted file mode 100644 index 2141e7ea7..000000000 --- a/common/docker-deps-clojure.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ - -RUN set -x \ - && cd /tmp \ - && lein deps \ - && true diff --git a/common/docker-deps-node.dockerfile b/common/docker-deps-node.dockerfile deleted file mode 100644 index a00baafff..000000000 --- a/common/docker-deps-node.dockerfile +++ /dev/null @@ -1,9 +0,0 @@ - -COPY package.json yarn.lock /tmp/ - -RUN set +x \ - && source $HOME/.nvm/nvm.sh \ - && cd /tmp \ - && yarn install \ - && sudo mv node_modules / \ - && true diff --git a/common/docker-deps-perl.dockerfile b/common/docker-deps-perl.dockerfile deleted file mode 100644 index b487c3f33..000000000 --- a/common/docker-deps-perl.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ - -RUN set -x \ - && sudo cpanm -n \ - Lingy \ - Regexp::Common \ - && true diff --git a/common/docker-deps-python.dockerfile b/common/docker-deps-python.dockerfile deleted file mode 100644 index e98d23521..000000000 --- a/common/docker-deps-python.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ - -RUN python3 -m venv /tmp/venv \ - && true diff --git a/common/docker-from-ubuntu.dockerfile b/common/docker-from-ubuntu.dockerfile deleted file mode 100644 index 09d44eac2..000000000 --- a/common/docker-from-ubuntu.dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG UBUNTU_VERSION -FROM ubuntu:$UBUNTU_VERSION -ARG DOCKER_UID=1000 -ARG DOCKER_GID=1000 -ARG GRAALVM_TAR=graalvm-jdk-21_linux-x64_bin.tar.gz -SHELL ["/bin/bash", "-c"] diff --git a/common/docker-install-graalvm.dockerfile b/common/docker-install-graalvm.dockerfile deleted file mode 100644 index a0a0abd70..000000000 --- a/common/docker-install-graalvm.dockerfile +++ /dev/null @@ -1,8 +0,0 @@ - -RUN set -x \ - && cd /tmp \ - && curl -o $GRAALVM_TAR \ - https://download.oracle.com/graalvm/21/latest/$GRAALVM_TAR \ - && tar xzf $GRAALVM_TAR \ - && mv graalvm-jdk-21.* graalvm-oracle-21 \ - && true diff --git a/common/docker-useradd.dockerfile b/common/docker-useradd.dockerfile deleted file mode 100644 index 0d1e71c9b..000000000 --- a/common/docker-useradd.dockerfile +++ /dev/null @@ -1,12 +0,0 @@ - -RUN set -x \ - && apt-get install -y sudo \ - && groupadd -g $DOCKER_GID user \ - && useradd -u $DOCKER_UID -g $DOCKER_GID user \ - && adduser user sudo \ - && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ - && mkdir -p /home/user \ - && chown -R user:user /home/user \ - && true - -USER user diff --git a/common/docker.mk b/common/docker.mk deleted file mode 100644 index e0f67304f..000000000 --- a/common/docker.mk +++ /dev/null @@ -1,38 +0,0 @@ -DOCKER_NAME := yamlscript-$(SUBDIR) -DOCKER_IMAGE := $(DOCKER_NAME):latest -DOCKER_HISTORY := $(YS_TMP)/$(DOCKER_NAME)-bash-history -export DOCKER_UID := $(shell id -u) -export DOCKER_GID := $(shell id -g) - -ifneq (,$(wildcard /.dockerenv)) - DOCKERENV := 1 -endif - -docker-build:: Dockerfile - cp $(COMMON)/project.clj .project.clj - docker build \ - --build-arg UBUNTU_VERSION=22.04 \ - --build-arg DOCKER_UID=$(DOCKER_UID) \ - --build-arg DOCKER_GID=$(DOCKER_GID) \ - --tag $(DOCKER_IMAGE) . - $(RM) $< .project.clj - -docker-test:: docker-build - docker run --rm -it \ - --volume $(ROOT):/host \ - --workdir /host/$(SUBDIR) \ - -u $$DOCKER_UID:$$DOCKER_GID \ - $(DOCKER_IMAGE) \ - make test v=$v - -docker-shell:: docker-build - touch $(DOCKER_HISTORY) - docker run --rm -it \ - --volume $(ROOT):/host \ - --volume $(DOCKER_HISTORY):/home/user/.bash_history \ - --workdir /host/$(SUBDIR) \ - --entrypoint /bin/bash \ - $(DOCKER_IMAGE) - -clean:: - $(RM) .project.clj diff --git a/common/java.mk b/common/java.mk index bc9997d8c..f95a424de 100644 --- a/common/java.mk +++ b/common/java.mk @@ -5,6 +5,10 @@ export JAVA_HOME := $(GRAALVM_HOME) export PATH := $(JAVA_HOME)/bin:$(PATH) +MVN ?= mvn +JAR ?= $(GRAALVM_HOME)/bin/jar +JAVAC ?= $(GRAALVM_HOME)/bin/javac + YAMLSCRIPT_JAVA_INSTALLED := \ $(MAVEN_REPOSITORY)/org/yamlscript/yamlscript/maven-metadata-local.xml @@ -31,3 +35,6 @@ endif $(YAMLSCRIPT_JAVA_INSTALLED): $(YAMLSCRIPT_JAVA_SRC) $(MAKE) -C $(ROOT)/java install + +$(JAVAC): $(GRAALVM_INSTALLED) +$(JAR): $(GRAALVM_INSTALLED) diff --git a/common/native.mk b/common/native.mk index 87b48eb29..62a98c696 100644 --- a/common/native.mk +++ b/common/native.mk @@ -6,10 +6,12 @@ NATIVE_OPTS := \ --initialize-at-build-time \ --enable-preview \ --enable-url-protocols=https \ + --emit=build-report \ -march=compatibility \ + -H:IncludeResources=SCI_VERSION \ + -H:IncludeResources='librapidyaml\.0\.8\.0\.so' \ -H:ReflectionConfigurationFiles=reflection.json \ -H:+ReportExceptionStackTraces \ - -H:IncludeResources=SCI_VERSION \ -H:Log=registerResource: \ -J-Dclojure.spec.skip-macros=true \ -J-Dclojure.compiler.direct-linking=true \ diff --git a/common/project.clj b/common/project.clj deleted file mode 100644 index 4eec95986..000000000 --- a/common/project.clj +++ /dev/null @@ -1,26 +0,0 @@ -;; This code is licensed under MIT license (See License for details) -;; Copyright 2023-2025 Ingy dot Net - -(defproject yamlscript/docker "0.1.95" - :description "Program in YAML — Code is Data" - :dependencies - [#__ - [borkdude/edamame "1.3.23"] - [clj-commons/clj-yaml "1.0.27"] - [commons-codec/commons-codec "1.11"] - [dev.weavejester/lein-cljfmt "0.11.2"] - [io.github.borkdude/lein-lein2deps "0.1.0"] - [lein-exec "0.3.7"] - [org.apache.httpcomponents/httpclient "4.5.13"] - [org.apache.httpcomponents/httpcore "4.4.15"] - [org.babashka/sci "0.8.41"] - [org.clojure/clojure "1.12.0"] - [org.clojure/data.json "2.4.0"] - [org.clojure/tools.gitlibs "1.0.100"] - [org.clojure/tools.reader "1.3.6"] - [org.flatland/ordered "1.15.11"] - [org.slf4j/slf4j-api "1.7.36"] - [org.snakeyaml/snakeyaml-engine "2.6"] - [pjstadig/humane-test-output "0.11.0"] - [reifyhealth/lein-git-down "0.4.1"] - #__]) diff --git a/common/reflection.json b/common/reflection.json index 3b64a1f6b..fc799578b 100644 --- a/common/reflection.json +++ b/common/reflection.json @@ -1,115 +1,125 @@ [ { - "name": "java.lang.AssertionError", + "name": "clojure.lang.Delay", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "java.lang.Class", - "allDeclaredConstructors": true, + "name": "clojure.lang.ExceptionInfo", "allPublicConstructors": true, - "allDeclaredMethods": true, + "allPublicFields": true, "allPublicMethods": true }, { - "name": "java.lang.Exception", + "name": "clojure.lang.LineNumberingPushbackReader", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "clojure.lang.Delay", + "name": "clojure.lang.RT", + "methods": [ + {"name": "aget"}, + {"name": "aset"}, + {"name": "aclone"} + ] + }, + { + "name": "java.io.StringReader", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name": "clojure.lang.ExceptionInfo", + "name": "java.io.StringWriter", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"clojure.lang.LineNumberingPushbackReader", + "name": "java.lang.AssertionError", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.String", + "name": "java.lang.Class", + "allDeclaredConstructors": true, + "allDeclaredMethods": true, "allPublicConstructors": true, - "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.io.StringReader", + "name": "java.lang.ClassLoader", + "allDeclaredConstructors": true, + "allDeclaredFields": true, + "allDeclaredMethods": true, "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.io.StringWriter", + "name": "java.lang.Double", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Double", + "name": "java.lang.Exception", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Integer", + "name": "java.lang.IllegalArgumentException", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Number", + "name": "java.lang.Integer", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.IllegalArgumentException", + "name": "java.lang.Math", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.System", + "name": "java.lang.Number", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Math", + "name": "java.lang.String", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, { - "name":"java.lang.Thread", + "name": "java.lang.System", "allPublicConstructors": true, "allPublicFields": true, "allPublicMethods": true }, - { - "name": "java.lang.reflect.AccessibleObject", - "methods": [{"name":"canAccess"}] + "name": "java.lang.Thread", + "allPublicConstructors": true, + "allPublicFields": true, + "allPublicMethods": true }, - { - "name": "clojure.lang.RT", + "name": "java.lang.reflect.AccessibleObject", "methods": [ - {"name": "aget"}, - {"name": "aset"}, - {"name": "aclone"} + {"name": "canAccess"} ] }, - - {"name": "[Ljava.lang.Double;"} + { + "name": "[Ljava.lang.Double;" + } ] diff --git a/common/vars-cli.mk b/common/vars-cli.mk new file mode 100644 index 000000000..f4c8728de --- /dev/null +++ b/common/vars-cli.mk @@ -0,0 +1,32 @@ +include $(COMMON)/vars-core.mk + +CLI_BIN := bin/ys-$(YS_VERSION) +CLI_SRC := \ + src/yamlscript/cli.clj \ + +CLI_BIN_BASH := bin/ys-sh-$(YS_VERSION) +CLI_BIN_BASH_SRC := share/ys-0.bash + +CLI_JAR := \ + target/uberjar/yamlscript.cli-$(YS_VERSION)-SNAPSHOT-standalone.jar + +CLI_JAR_DEPS := \ + $(LEIN) \ + $(CORE_INSTALLED) \ + $(CLI_SRC) \ + +CLI_DEPS := \ + $(CLI_BIN) \ + $(CLI_BIN_BASH) \ + +ifdef YS_NATIVE_BUILD_STATIC +ifeq (true,$(IS_LINUX)) +ifeq (true,$(IS_INTEL)) +CLI_DEPS := $(MUSL_GCC) $(CLI_DEPS) +NATIVE_OPTS += \ + -H:CCompilerOption=-Wl,-z,stack-size=2097152 \ + --static \ + --libc=musl +endif +endif +endif diff --git a/common/vars-core.mk b/common/vars-core.mk new file mode 100644 index 000000000..f96fbd848 --- /dev/null +++ b/common/vars-core.mk @@ -0,0 +1,10 @@ +include $(COMMON)/vars-rapidyaml.mk + +CORE_DIR := $(ROOT)/core +CORE_JAR := $(CORE_DIR)/target/core-$(YAMLSCRIPT_VERSION)-standalone.jar +CORE_DEPS := $(LEIN) $(RAPIDYAML_INSTALLED) + +CORE_INSTALLED := \ + $(MAVEN_REPOSITORY)/yamlscript/core/$(YAMLSCRIPT_VERSION) +CORE_INSTALLED := \ + $(CORE_INSTALLED)/core-$(YAMLSCRIPT_VERSION).jar diff --git a/common/vars-libys.mk b/common/vars-libys.mk new file mode 100644 index 000000000..399d587e1 --- /dev/null +++ b/common/vars-libys.mk @@ -0,0 +1,33 @@ +LIBYS_DIR := $(ROOT)/libyamlscript +LIBYS_LIB := $(LIBYS_DIR)/lib + +export LD_LIBRARY_PATH := $(LIBYS_LIB):$(LD_LIBRARY_PATH) +export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH) + +LIBYS_SO_NAME := $(LIBYS_LIB)/libyamlscript +LIBYS_SO_FQNP := $(LIBYS_SO_NAME).$(SO).$(YAMLSCRIPT_VERSION) +LIBYS_SO_BASE := $(LIBYS_LIB)/libyamlscript.$(SO) +LIBYS_SO_APIP := $(LIBYS_SO_BASE).$(API_VERSION) +LIBYS_SO_VERS := $(LIBYS_LIB)/libyamlscript.$(YAMLSCRIPT_VERSION).$(SO) + +LIBYS_DEPS := \ + $(LIBYS_SO_FQNP) \ + +LIBYS_JAR := \ + $(LIBYS_LIB)/target/libyamlscript-$(YAMLSCRIPT_VERSION)-standalone.jar + +LIBYS_INSTALLED := \ + $(MAVEN_REPOSITORY)/org/yamlscript/yamlscript/$(YAMLSCRIPT_VERSION) +LIBYS_INSTALLED := \ + $(LIBYS_INSTALLED)/yamlscript-$(YAMLSCRIPT_VERSION).jar + +LIBYS_JAR_PATH := \ + target/libyamlscript-$(YAMLSCRIPT_VERSION)-standalone.jar + +LIBYS_SOURCES := \ + src/libyamlscript/core.clj \ + src/libyamlscript/API.java \ + +LIBYS_HEADERS := \ + $(LIBYS_LIB)/graal_isolate.h \ + $(LIBYS_SO_NAME).$(YAMLSCRIPT_VERSION).h \ diff --git a/common/vars-rapidyaml.mk b/common/vars-rapidyaml.mk new file mode 100644 index 000000000..43d3b2cde --- /dev/null +++ b/common/vars-rapidyaml.mk @@ -0,0 +1,28 @@ +RAPIDYAML_DIR := $(ROOT)/rapidyaml +NATIVE_DIR := $(RAPIDYAML_DIR)/native + +export LD_LIBRARY_PATH := $(NATIVE_DIR):$(LD_LIBRARY_PATH) +export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH) + +RAPIDYAML_VERSION := 0.10.0 +RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION) +RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml +RAPIDYAML_BUILD_TYPE ?= Release +RAPIDYAML_DBG ?= 0 +RAPIDYAML_TIMED ?= 1 +RAPIDYAML_JAVA := \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/Rapidyaml.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/NativeLibLoader.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/Evt.java \ + $(RAPIDYAML_DIR)/src/main/java/org/rapidyaml/YamlParseErrorException.java +RAPIDYAML_NAME := ysparse +RAPIDYAML_LIBNAME := lib$(RAPIDYAML_NAME) +RAPIDYAML_JNI_H := $(NATIVE_DIR)/org_rapidyaml_Rapidyaml.h +RAPIDYAML_SO_NAME := $(RAPIDYAML_LIBNAME).$(RAPIDYAML_VERSION).$(SO) +RAPIDYAML_SO := $(NATIVE_DIR)/$(RAPIDYAML_SO_NAME) +RAPIDYAML_LIB := $(NATIVE_DIR)/$(RAPIDYAML_LIBNAME).$(DOTLIB) +RAPIDYAML_JAR := $(RAPIDYAML_DIR)/target/rapidyaml-$(RAPIDYAML_VERSION).jar +RAPIDYAML_INSTALLED := \ + $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION) +RAPIDYAML_INSTALLED := \ + $(RAPIDYAML_INSTALLED)/rapidyaml-$(RAPIDYAML_VERSION).jar diff --git a/common/vars.mk b/common/vars.mk index 154eab278..c241abb06 100644 --- a/common/vars.mk +++ b/common/vars.mk @@ -27,6 +27,7 @@ export YAMLSCRIPT_ROOT ?= $(ROOT) export API_VERSION := 0 export YS_VERSION := $(shell grep '^version:' $(ROOT)/Meta | cut -d' ' -f2) +YAMLSCRIPT_VERSION := $(YS_VERSION) ifdef v export TEST_VERBOSE := 1 @@ -48,12 +49,15 @@ ifneq (,$(findstring linux,$(ostype))) GCC := gcc -std=gnu99 -fPIC -shared SO := so DY := + DOTLIB := a else ifneq (,$(findstring darwin,$(ostype))) IS_MACOS := true GCC := gcc -dynamiclib SO := dylib DY := DY + DOTLIB := a else + DOTLIB := lib $(error Unsupported OSTYPE: $(ostype)) endif @@ -82,16 +86,6 @@ CURL := $(shell command -v curl) TIME := time -p -LIBYAMLSCRIPT_DIR := $(ROOT)/libyamlscript/lib -LIBRARY_PATH := $(LIBYAMLSCRIPT_DIR) -export $(DY)LD_LIBRARY_PATH := $(LIBRARY_PATH) -export LD_LIBRARY_PATH := $(LIBRARY_PATH) -LIBYAMLSCRIPT_SO_NAME := $(LIBYAMLSCRIPT_DIR)/libyamlscript -LIBYAMLSCRIPT_SO_FQNP := $(LIBYAMLSCRIPT_SO_NAME).$(SO).$(YS_VERSION) -LIBYAMLSCRIPT_SO_BASE := $(LIBYAMLSCRIPT_DIR)/libyamlscript.$(SO) -LIBYAMLSCRIPT_SO_APIP := $(LIBYAMLSCRIPT_SO_BASE).$(API_VERSION) -LIBYAMLSCRIPT_SO_VERS := $(LIBYAMLSCRIPT_DIR)/libyamlscript.$(YS_VERSION).$(SO) - ifeq (true,$(IS_ROOT)) PREFIX ?= /usr/local else @@ -165,6 +159,7 @@ GRAALVM_DOWNLOAD := $(YS_TMP)/$(GRAALVM_TAR) GRAALVM_INSTALLED := $(GRAALVM_HOME)/release GRAALVM_O ?= 1 +# qbm is Quick Build Mode ifdef qbm GRAALVM_O := b endif @@ -174,7 +169,7 @@ endif # Set MAVEN variables: #------------------------------------------------------------------------------ -MAVEN_VER := 3.9.6 +MAVEN_VER := 3.9.11 MAVEN_SRC := https://dlcdn.apache.org/maven/maven-3/$(MAVEN_VER)/binaries MAVEN_TAR := apache-maven-$(MAVEN_VER)-bin.tar.gz MAVEN_URL := $(MAVEN_SRC)/$(MAVEN_TAR) @@ -221,6 +216,14 @@ RELEASE_LYS_NAME := libyamlscript-$(YS_VERSION)-$(GRAALVM_ARCH) RELEASE_LYS_TAR := $(RELEASE_LYS_NAME).tar.xz +#------------------------------------------------------------------------------ +# Programs +#------------------------------------------------------------------------------ + +GIT ?= git +CMAKE ?= cmake + + #------------------------------------------------------------------------------ default:: diff --git a/core/Makefile b/core/Makefile index a6d28f544..fe22a1403 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1,26 +1,30 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk +include $(COMMON)/vars-core.mk +include $(COMMON)/vars-rapidyaml.mk export PATH := $(ROOT)/core/bin:$(PATH) #------------------------------------------------------------------------------ -build:: - -install test:: $(LEIN) build - $< $@ - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ + +test:: $(CORE_DEPS) + $(LEIN) $@ + +install: $(CORE_INSTALLED) + +repl-deps:: $(RAPIDYAML_INSTALLED) + +$(CORE_INSTALLED): $(CORE_JAR) $(RAPIDYAML_INSTALLED) + $(LEIN) install + touch $@ + +$(CORE_JAR): $(CORE_DEPS) + $(LEIN) uberjar + touch $@ + +$(RAPIDYAML_INSTALLED): + $(MAKE) -C $(RAPIDYAML_DIR) $@ clean:: $(RM) pom.xml diff --git a/core/deps.edn b/core/deps.edn index cbb5af22d..2aa96efb0 100644 --- a/core/deps.edn +++ b/core/deps.edn @@ -5,6 +5,7 @@ org.clojure/data.json {:mvn/version "2.4.0"}, clj-commons/clj-yaml {:mvn/version "1.0.27"}, org.flatland/ordered {:mvn/version "1.15.11"}, + org.rapidyaml/rapidyaml {:mvn/version "0.8.0"}, org.snakeyaml/snakeyaml-engine {:mvn/version "2.7"}, babashka/babashka.pods {:mvn/version "0.2.0"}, babashka/fs {:mvn/version "0.5.20"}, diff --git a/core/project.clj b/core/project.clj index ed58e1860..414c061b0 100644 --- a/core/project.clj +++ b/core/project.clj @@ -22,6 +22,7 @@ [org.clojure/data.json "2.4.0"] [clj-commons/clj-yaml "1.0.27"] [org.flatland/ordered "1.15.11"] + [org.rapidyaml/rapidyaml "0.8.0"] [org.snakeyaml/snakeyaml-engine "2.7"] [babashka/babashka.pods "0.2.0"] [babashka/fs "0.5.20"] diff --git a/core/src/yamlscript/parser.clj b/core/src/yamlscript/parser.clj index 00e2af5b4..4eabd34a1 100644 --- a/core/src/yamlscript/parser.clj +++ b/core/src/yamlscript/parser.clj @@ -4,14 +4,15 @@ ;; The yamlscript.parser is responsible for parsing YAML into a sequence of ;; event objects. -;; TODO -;; - switch from snakeyaml to libfyaml (ffi) - (ns yamlscript.parser (:require + [clojure.string :as str] [yamlscript.common]) (:import (java.util Optional) + (java.nio ByteBuffer) + (java.nio.charset StandardCharsets) + (org.rapidyaml Evt Rapidyaml) (org.snakeyaml.engine.v2.api LoadSettings) (org.snakeyaml.engine.v2.api.lowlevel Parse) (org.snakeyaml.engine.v2.exceptions Mark) @@ -29,21 +30,22 @@ SequenceEndEvent)) (:refer-clojure)) -(declare ys-event) +(declare parse-fn) + +(defn TIMER [] (System/getenv "YS_TIMER")) (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n") (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(") + (defn parse "Parse a YAML string into a sequence of event objects." [yaml-string] - (let [parser (new Parse (.build (LoadSettings/builder))) - has-code-mode-shebang (or (re-find shebang-ys yaml-string) + (let [has-code-mode-shebang (or + (re-find shebang-ys yaml-string) (re-find shebang-bash yaml-string)) - events (->> yaml-string - (.parseString parser) - (map ys-event) - (remove nil?) - rest) + events (if (TIMER) + (time ((parse-fn) yaml-string)) + ((parse-fn) yaml-string)) [first-event & rest-events] events first-event-tag (:! first-event) first-event (if (and has-code-mode-shebang @@ -56,10 +58,20 @@ events (cons first-event rest-events)] (remove nil? events))) -(defn parse-test-case [yaml-string] - (->> yaml-string - parse - (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) +(declare snake-event) + +;; +;; SnakeYAML Parser +;; + +;; TODO - Set bigger buffer size in scanner class +(defn parse-snakeyaml [yaml-string] + (let [parser (new Parse (.build (LoadSettings/builder)))] + (->> yaml-string + (.parseString parser) + (map snake-event) + (remove nil?) + rest))) ;; ;; Functions to turn Java event objects into Clojure objects @@ -132,16 +144,159 @@ (let [obj (event-obj event)] (assoc obj :* (str (. event getAlias))))) -(defmulti ys-event class) -(defmethod ys-event DocumentStartEvent [event] (doc-start event)) -(defmethod ys-event DocumentEndEvent [event] (doc-end event)) -(defmethod ys-event MappingStartEvent [event] (map-start event)) -(defmethod ys-event MappingEndEvent [event] (map-end event)) -(defmethod ys-event SequenceStartEvent [event] (seq-start event)) -(defmethod ys-event SequenceEndEvent [event] (seq-end event)) -(defmethod ys-event ScalarEvent [event] (scalar-val event)) -(defmethod ys-event AliasEvent [event] (alias-val event)) -(defmethod ys-event :default [_] nil) +(defmulti snake-event class) +(defmethod snake-event DocumentStartEvent [event] (doc-start event)) +(defmethod snake-event DocumentEndEvent [event] (doc-end event)) +(defmethod snake-event MappingStartEvent [event] (map-start event)) +(defmethod snake-event MappingEndEvent [event] (map-end event)) +(defmethod snake-event SequenceStartEvent [event] (seq-start event)) +(defmethod snake-event SequenceEndEvent [event] (seq-end event)) +(defmethod snake-event ScalarEvent [event] (scalar-val event)) +(defmethod snake-event AliasEvent [event] (alias-val event)) +(defmethod snake-event :default [_] nil) + +;; +;; RapidYAML Parser +;; + +(defn event-type [mask] + (condp = (bit-and mask 2r11111111111) + Evt/BSTR nil + Evt/ESTR nil + Evt/BDOC "+DOC" + Evt/EDOC "-DOC" + Evt/BMAP "+MAP" + Evt/EMAP "-MAP" + Evt/BSEQ "+SEQ" + Evt/ESEQ "-SEQ" + Evt/SCLR "=VAL" + Evt/ALIA "=ALI" + nil)) + +(defmacro flag? [flag mask] + `(pos? (bit-and ~mask (. Evt ~flag)))) + +(defn get-skey [mask] + (condp = (bit-and mask 2r111110000000000000000) + Evt/PLAI := + Evt/SQUO :' + Evt/DQUO :$ + Evt/LITL :| + Evt/FOLD :> + nil)) + +(defn parse-rapidyaml-arr [^String yaml-string] + (rest + (let [parser ^Rapidyaml (new Rapidyaml) + _ (when (TIMER) + (.timingEnabled parser true)) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array 5) + needed (.parseYsToEvt parser buffer masks) + buffer (.getBytes yaml-string StandardCharsets/UTF_8) + masks (int-array needed) + _ (.parseYsToEvt parser buffer masks) + ;; TODO: aget slow? + ;; https://stackoverflow.com/questions/10133094/clojure-why-is-aget-so-slow + get-str (fn [i] + (let [off (aget masks (inc i)) + len (aget masks (+ i 2))] + (String. buffer off len StandardCharsets/UTF_8)))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (aget masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + +(defn parse-rapidyaml-buf [^String yaml-string] + (rest + (let [parser (new Rapidyaml) + _ (when (TIMER) + (.timingEnabled parser true)) + srcbytes (.getBytes yaml-string StandardCharsets/UTF_8) + srcbuffer (ByteBuffer/allocateDirect (alength srcbytes)) + _ (.put srcbuffer srcbytes) + masks (Rapidyaml/mkIntBuffer 5) + needed (.parseYsToEvtBuf parser srcbuffer masks) + _ (.position srcbuffer 0) + _ (.put srcbuffer srcbytes) + masks (Rapidyaml/mkIntBuffer needed) + _ (.parseYsToEvtBuf parser srcbuffer masks) + get-str (fn [i] + (let [off (.get masks ^int (inc i)) + len (.get masks ^int (+ i 2))] + (.toString + (.decode StandardCharsets/UTF_8 + (.slice srcbuffer off len)))))] + + (loop [i 0, tag nil, anchor nil, events []] + (if (< i needed) + (let [mask (.get masks i) + type (event-type mask) + ; _ (WWW (Integer/toString mask 2) type) + sval (when (flag? HAS_STR mask) (get-str i)) + tag (if (flag? TAG_ mask) sval tag) + anchor (if (flag? ANCH mask) sval anchor) + event (when type + (let [event {:+ type} + event (if (flag? FLOW mask) + (assoc event :flow true) event) + event (if anchor (assoc event :& anchor) event) + event (if tag + (let [tag (str/replace tag + #"^!!" + "tag:yaml.org,2002:")] + (assoc event :! tag)) event) + event (if sval (assoc event + (get-skey mask) sval) event) + event (if (= type "=ALI") + {:+ "=ALI" :* sval} + event)] + event)) + events (if event (conj events event) events) + i (+ i (if sval 3 1))] + (if event + (recur i nil nil events) + (recur i tag anchor events))) + events))))) + + +(defn parse-fn [] + (if-let [parser-name (System/getenv "YS_PARSER")] + (condp = parser-name + "" parse-snakeyaml + "snake" parse-snakeyaml + "rapid" parse-rapidyaml-buf + "rapid-buf" parse-rapidyaml-buf + "rapid-arr" parse-rapidyaml-arr + (die "Unknown YS_PARSER value: " parser-name)) + parse-snakeyaml)) (comment ) diff --git a/core/test/yamlscript/parser_test.clj b/core/test/yamlscript/parser_test.clj index 29420b7d8..ef04eec4b 100644 --- a/core/test/yamlscript/parser_test.clj +++ b/core/test/yamlscript/parser_test.clj @@ -8,6 +8,11 @@ [yamlscript.parser :as parser] [yamltest.core :as test])) +(defn parse-test-case [yaml-string] + (->> yaml-string + parser/parse + (remove (fn [ev] (= "DOC" (subs (:+ ev) 1)))))) + (test/load-yaml-test-files ["test/compiler-stack.yaml" "test/resolver.yaml" @@ -16,7 +21,7 @@ :test (fn [test] (->> test :yamlscript - parser/parse-test-case + parse-test-case (map pr-str) (map #(subs %1 4 (dec (count %1)))))) :want (fn [test] diff --git a/go/Makefile b/go/Makefile index eb35bbcbc..3b673ca02 100644 --- a/go/Makefile +++ b/go/Makefile @@ -42,7 +42,7 @@ export PATH := $(GOBIN):$(PATH) #------------------------------------------------------------------------------ -build:: build-doc $(LIBYAMLSCRIPT_SO_FQNP) $(GO_INSTALLED) +build:: build-doc $(LIBYS_SO_FQNP) $(GO_INSTALLED) build-doc:: ReadMe.md diff --git a/java/Makefile b/java/Makefile index 1a265fe08..ed27e36a5 100644 --- a/java/Makefile +++ b/java/Makefile @@ -13,10 +13,10 @@ MVN_COMMANDS := \ #------------------------------------------------------------------------------ -$(MVN_COMMANDS):: $(JAVA_INSTALLED) $(LIBYAMLSCRIPT_SO_FQNP) +$(MVN_COMMANDS):: $(JAVA_INSTALLED) $(LIBYS_SO_FQNP) mvn $@ -release: $(JAVA_INSTALLED) $(LIBYAMLSCRIPT_SO_FQNP) test package +release: $(JAVA_INSTALLED) $(LIBYS_SO_FQNP) test package ifndef n $(error 'make $@' needs the n variable set to the new version) endif diff --git a/julia/Makefile b/julia/Makefile index 513a2a139..ca1b9715c 100644 --- a/julia/Makefile +++ b/julia/Makefile @@ -11,7 +11,7 @@ build:: build-doc build-doc:: ReadMe.md ifdef JULIA -test:: $(LIBYAMLSCRIPT_SO_FQNP) deps +test:: $(LIBYS_SO_FQNP) deps $(JULIA) --project=. test/runtests.jl deps:: diff --git a/libyamlscript/Makefile b/libyamlscript/Makefile index 2b58ba6da..46e66453e 100644 --- a/libyamlscript/Makefile +++ b/libyamlscript/Makefile @@ -1,54 +1,34 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk include $(COMMON)/native.mk - -LIBYAMLSCRIPT_JAR_PATH := target/libyamlscript-$(YS_VERSION)-standalone.jar - -LYS_SOURCES := \ - src/libyamlscript/core.clj \ - src/libyamlscript/API.java \ - -HEADERS := \ - $(LIBYAMLSCRIPT_DIR)/graal_isolate.h \ - $(LIBYAMLSCRIPT_SO_NAME).$(YS_VERSION).h \ - -BUILD_TARGETS := \ - $(LIBYAMLSCRIPT_SO_FQNP) \ - $(HEADERS) \ - -# Avoid rebuild in ephemerally created Docker container. -ifdef DOCKERENV -ifneq (,$(wildcard $(LIBYAMLSCRIPT_SO_FQNP))) -LIBYAMLSCRIPT_JAR_PATH := -endif -endif +include $(COMMON)/vars-core.mk +include $(COMMON)/vars-libys.mk #------------------------------------------------------------------------------ -build:: $(BUILD_TARGETS) +build:: $(LIBYS_DEPS) -jar: $(LIBYAMLSCRIPT_JAR_PATH) +# jar: $(LIBYS_JAR_PATH) -install:: $(BUILD_TARGETS) +install:: $(LIBYS_DEPS) mkdir -p $(PREFIX)/include/ - install -m 644 $(HEADERS) $(PREFIX)/include/ + install -m 644 $(LIBYS_HEADERS) $(PREFIX)/include/ mkdir -p $(PREFIX)/lib/ install -m 644 $< $(PREFIX)/lib/ - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_BASE)) - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_APIP)) - ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYAMLSCRIPT_SO_VERS)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_BASE)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_APIP)) + ln -fs $(notdir $<) $(PREFIX)/lib/$(notdir $(LIBYS_SO_VERS)) realclean:: $(RM) -r lib/ $(RM) src/libyamlscript/Core.class -test:: $(LIBYAMLSCRIPT_SO_FQNP) +test:: $(LIBYS_SO_FQNP) ls -lh lib -repl-deps:: $(LIBYAMLSCRIPT_JAR_PATH) +repl-deps:: $(LIBYS_JAR_PATH) -$(LIBYAMLSCRIPT_SO_FQNP): $(LIBYAMLSCRIPT_JAR_PATH) +$(LIBYS_SO_FQNP): $(LIBYS_JAR_PATH) ifneq (true,$(LIBZ)) $(error *** The 'libz.$(SO)' library is required by native-image but not installed) endif @@ -60,31 +40,17 @@ endif \ --shared \ -jar $< \ - -o $(LIBYAMLSCRIPT_SO_NAME) - mv $(LIBYAMLSCRIPT_SO_BASE) $@ - mv $(LIBYAMLSCRIPT_SO_NAME).h $(LIBYAMLSCRIPT_SO_NAME).$(YS_VERSION).h - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_APIP) - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_VERS) - ln -fs $(notdir $@) $(LIBYAMLSCRIPT_SO_BASE) - -ifeq (true,$(IS_ROOT)) -$(LIBYAMLSCRIPT_JAR_PATH): -ifeq (true,$(IS_ROOT)) - $(error *** YS 'make build' must be run as non-root user) -endif -else -$(LIBYAMLSCRIPT_JAR_PATH): $(LEIN) $(JAVA_INSTALLED) $(YAMLSCRIPT_CORE_INSTALLED) $(LYS_SOURCES) - $< uberjar -endif - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ + -o $(LIBYS_SO_NAME) + mv $(LIBYS_SO_BASE) $@ + mv $(LIBYS_SO_NAME).h $(LIBYS_SO_NAME).$(YAMLSCRIPT_VERSION).h + ln -fs $(notdir $@) $(LIBYS_SO_APIP) + ln -fs $(notdir $@) $(LIBYS_SO_VERS) + ln -fs $(notdir $@) $(LIBYS_SO_BASE) + touch $@ + +$(LIBYS_JAR_PATH): $(CORE_INSTALLED) + $(LEIN) uberjar + touch $@ + +$(CORE_INSTALLED): + $(MAKE) -C $(CORE_DIR) $@ diff --git a/libyamlscript/src/libyamlscript/API.java b/libyamlscript/src/libyamlscript/API.java index 008adf8ef..060c99f33 100644 --- a/libyamlscript/src/libyamlscript/API.java +++ b/libyamlscript/src/libyamlscript/API.java @@ -33,7 +33,7 @@ public final class API { } public static void debug(String s) { - if (System.getenv("LIBYAMLSCRIPT_DEBUG") != null) { + if (System.getenv("YS_DEBUG") != null) { System.err.println(s); } } diff --git a/libyamlscript/src/libyamlscript/core.clj b/libyamlscript/src/libyamlscript/core.clj index 156a4f084..a5f5f17c3 100644 --- a/libyamlscript/src/libyamlscript/core.clj +++ b/libyamlscript/src/libyamlscript/core.clj @@ -49,7 +49,7 @@ :trace (get-in err [:trace])}})) (defn debug [& msg] - (when (System/getenv "LIBYAMLSCRIPT_DEBUG") + (when (System/getenv "YS_DEBUG") (binding [*out* *err*] (apply println msg)))) diff --git a/nodejs/Makefile b/nodejs/Makefile index d95110448..4ad6032c6 100644 --- a/nodejs/Makefile +++ b/nodejs/Makefile @@ -27,7 +27,7 @@ build:: build-doc build-doc:: ReadMe.md -test:: update $(LIBYAMLSCRIPT_SO_FQNP) +test:: update $(LIBYS_SO_FQNP) @printf '%.0s-' {1..80}; echo $(strace) node test/test.js diff --git a/perl/Makefile b/perl/Makefile index 29e9710f5..caf9fdc58 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -55,13 +55,3 @@ $(ALIEN_DIST_DIR): $(LOCAL_LIB): cpanm -L $@ -n $(CPAN_DEPS) - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ diff --git a/python/Makefile b/python/Makefile index 249694c72..9e28964b3 100644 --- a/python/Makefile +++ b/python/Makefile @@ -2,11 +2,7 @@ include ../common/base.mk include $(COMMON)/binding.mk include $(COMMON)/python.mk -ifdef DOCKERENV - PYTHON_VENV := $(YS_TMP)/venv -else - PYTHON_VENV := $(ROOT)/python/.venv -endif +PYTHON_VENV := $(ROOT)/python/.venv ACTIVATE := source $(PYTHON_VENV)/bin/activate @@ -33,7 +29,7 @@ test-pytest: $(PYTHON_VENV) $(ACTIVATE) && \ pytest $${TEST_VERBOSE:+'-v'} test/*.py -test-ffi: $(LIBYAMLSCRIPT_SO_FQNP) +test-ffi: $(LIBYS_SO_FQNP) $(ACTIVATE) && \ $(PYTHON) \ -c 'import yamlscript; \ @@ -94,17 +90,3 @@ MANIFEST.in: grep -B999 '## Features' | \ head -n-2 \ > $@ - -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ diff --git a/python/ReadMe.md b/python/ReadMe.md index 2677f1b95..c5eead29a 100644 --- a/python/ReadMe.md +++ b/python/ReadMe.md @@ -126,7 +126,7 @@ $ python prog.py You can install this module like any other Python module: ```bash -$ pip install yamlscript +pip install yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -134,7 +134,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/rapidyaml/Makefile b/rapidyaml/Makefile new file mode 100644 index 000000000..f1b79ff19 --- /dev/null +++ b/rapidyaml/Makefile @@ -0,0 +1,75 @@ +include ../common/base.mk +include $(COMMON)/clojure.mk +include $(COMMON)/java.mk +include $(COMMON)/vars-rapidyaml.mk + +RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class) +RAPIDYAML_CLASSES := \ + $(RAPIDYAML_CLASS:$(ROOT)/rapidyaml/src/main/java/%=-C src/main/java %) + + +#------------------------------------------------------------------------------ +default:: + +build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RAPIDYAML_JAR) + @: + +jar: $(RAPIDYAML_JAR) + +jar-test: $(RAPIDYAML_JAR) + java -jar $< + +# install:: $(RAPIDYAML_INSTALLED) + +test:: build $(JAVA_INSTALLED) + $(MAKE) -C native $@ + YS_TESTING=1 $(MVN) $@ + +test-x: build $(JAVA_INSTALLED) + $(MAKE) -C native test + YS_TESTING=1 $(MVN) -X -e test + +clean:: + $(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) + $(RM) -r reports/ target/ + $(MAKE) -C native $@ + +realclean:: clean + $(MAKE) -C native $@ + $(RM) -r $(HOME)/.m2/repository/org/rapidyaml/ + +sysclean:: + $(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml + +jni: $(RAPIDYAML_JNI_H) + + +#------------------------------------------------------------------------------ +$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H) + $(MAKE) -C native $@ + +# XXX Probably don't need this: +$(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H) + $(MAKE) -C native $@ + +$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) +ifdef YS_MAVEN_TEST_SKIP + $(MVN) install -Dmaven.test.skip +else + YS_TESTING=1 $(MVN) install +endif + touch $@ + +$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) + $(JAR) -cf $@ $(RAPIDYAML_CLASSES) + $(JAR) -uf $@ -C $(ROOT)/rapidyaml/native/ $(RAPIDYAML_SO_NAME) + $(JAR) -umf manifest.txt $@ + +$(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED) + @# this doesn't work: + @#$(JAVAC) $< + @# ... but this does: + $(JAVAC) $(RAPIDYAML_JAVA) + +$(RAPIDYAML_JNI_H): $(RAPIDYAML_JAVA) + $(MAKE) -C native $@ diff --git a/rapidyaml/manifest.txt b/rapidyaml/manifest.txt new file mode 100644 index 000000000..280e5a906 --- /dev/null +++ b/rapidyaml/manifest.txt @@ -0,0 +1 @@ +Main-Class: org.rapidyaml.Rapidyaml diff --git a/rapidyaml/native/.gitignore b/rapidyaml/native/.gitignore new file mode 100644 index 000000000..2a816d627 --- /dev/null +++ b/rapidyaml/native/.gitignore @@ -0,0 +1,6 @@ +/rapidyaml/ +/_build/ +/build/ +/librapidyaml.* +/.cache +/compile_commands.json diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt new file mode 100644 index 000000000..e70ae2a68 --- /dev/null +++ b/rapidyaml/native/CMakeLists.txt @@ -0,0 +1,80 @@ +cmake_minimum_required(VERSION 3.12) +project(ysparse + DESCRIPTION "ysparse: rapidyaml -> yamlscript" + HOMEPAGE_URL "https://github.com/biojppm/rapidyaml -> https://github.com/yaml/yamlscript" + LANGUAGES CXX) + +find_package(JNI REQUIRED) + +option(YSPARSE_TIMED "add timings to sections" OFF) +option(YSPARSE_DBG "enable debug logs" OFF) + +if(UNIX) + set(CMAKE_SHARED_LIBRARY_SUFFIX .so) +endif() + + +add_library(ysparse + # + # JNI bridge + org_rapidyaml_Rapidyaml.h + org_rapidyaml_Rapidyaml.cpp + # + # ysparse files + ysparse_common.hpp + ysparse_evt.hpp + ysparse_evt.cpp + # + # files required from rapidyaml + rapidyaml/src/c4/yml/common.hpp + rapidyaml/src/c4/yml/common.cpp + rapidyaml/src/c4/yml/node_type.hpp + rapidyaml/src/c4/yml/node_type.cpp + rapidyaml/src/c4/yml/tag.cpp + rapidyaml/src_extra/c4/yml/extra/event_handler_ints.hpp + rapidyaml/src_extra/c4/yml/extra/event_handler_ints.cpp + # + # files required from rapidyaml/ext/c4core + rapidyaml/ext/c4core/src/c4/substr.hpp + rapidyaml/ext/c4core/src/c4/base64.hpp + rapidyaml/ext/c4core/src/c4/base64.cpp + rapidyaml/ext/c4core/src/c4/error.hpp + rapidyaml/ext/c4core/src/c4/error.cpp + rapidyaml/ext/c4core/src/c4/language.hpp + rapidyaml/ext/c4core/src/c4/language.cpp + rapidyaml/ext/c4core/src/c4/utf.hpp + rapidyaml/ext/c4core/src/c4/utf.cpp +) +target_include_directories(ysparse PUBLIC + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src_extra + ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src +) +target_compile_definitions(ysparse PUBLIC + RYML_WITH_TAB_TOKENS + RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS + $<$:YSPARSE_TIMED> + $<$:RYML_DBG> +) +set_target_properties(ysparse PROPERTIES CXX_STANDARD 17) + +target_include_directories(ysparse PUBLIC ${JNI_INCLUDE_DIRS}) + +add_executable(ysparse-test + ysparse_test.cpp + rapidyaml/src_extra/c4/yml/extra/ints_utils.hpp + rapidyaml/src_extra/c4/yml/extra/ints_utils.cpp + rapidyaml/src_extra/c4/yml/extra/scalar.cpp +) +target_link_libraries(ysparse-test ysparse) +add_custom_target(ysparse-test-run + DEPENDS ysparse-test + COMMAND $ + COMMENT "running C++ tests" +) +add_custom_target(ysparse-test-run-timing + DEPENDS ysparse-test + COMMAND $ --timing + COMMENT "running C++ tests, with timing" +) diff --git a/rapidyaml/native/Makefile b/rapidyaml/native/Makefile new file mode 100644 index 000000000..e1d8ebafe --- /dev/null +++ b/rapidyaml/native/Makefile @@ -0,0 +1,86 @@ +include ../../common/base.mk +include $(COMMON)/clojure.mk +include $(COMMON)/java.mk +include $(COMMON)/python.mk +include $(COMMON)/vars-rapidyaml.mk + +# TODO change to static library! +# https://www.graalvm.org/latest/reference-manual/native-image/guides/build-static-executables/ +# https://www.blog.akhil.cc/static-jni +# https://stackoverflow.com/questions/24493337/linking-static-library-with-jni + +THIS_DIR := $(shell pwd) +BDIR := $(THIS_DIR)/build/$(RAPIDYAML_BUILD_TYPE)-timed$(RAPIDYAML_TIMED)-dbg$(RAPIDYAML_DBG) + +RAPIDYAML_DEPS := \ + Makefile \ + CMakeLists.txt \ + $(JAVA_HOME) \ + $(RAPIDYAML_JNI_H) \ + $(wildcard ./*pp) \ + +CMK_ENV ?= +CMK_FLAGS_EXTRA ?= +CMK_FLAGS := \ + -D CMAKE_BUILD_TYPE=$(RAPIDYAML_BUILD_TYPE) \ + -D YSPARSE_TIMED=$(RAPIDYAML_TIMED) \ + -D YSPARSE_DBG=$(RAPIDYAML_DBG) \ + -D CMAKE_EXPORT_COMPILE_COMMANDS=ON + + +#------------------------------------------------------------------------------ +default:: + +cfg:: cfg-static cfg-shared +build:: build-static build-shared +test: test-static test-shared + +rapidyaml: + mkdir -p $@ + $(GIT) -C $@ init -q . + $(GIT) -C $@ remote add origin $(RAPIDYAML_REPO) + $(GIT) -C $@ fetch origin $(RAPIDYAML_TAG) + $(GIT) -C $@ reset --hard FETCH_HEAD + $(GIT) -C $@ submodule update --init --recursive + +clean:: + $(RM) librapidyaml.* + $(RM) -r _build + $(RM) -r rapidyaml-install + +realclean:: clean + $(RM) -r rapidyaml + +jni: $(RAPIDYAML_JNI_H) + +jnicheck: jni + $(GIT) diff --exit-code $(RAPIDYAML_JNI_H) + + +#------------------------------------------------------------------------------ + +build-static: $(RAPIDYAML_LIB) +cfg-static: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-static $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=OFF +test-static: build-static + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target $(RAPIDYAML_NAME)-test-run +test-static-timing: build-static + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --target $(RAPIDYAML_NAME)-test-run-timing +$(RAPIDYAML_LIB): cfg-static $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-static --verbose --parallel --target $(RAPIDYAML_NAME) + cp -fv $(BDIR)-static/*.a $@ + +build-shared: $(RAPIDYAML_SO) +cfg-shared: rapidyaml + $(CMK_ENV) $(CMAKE) -S . -B $(BDIR)-shared $(CMK_FLAGS) $(CMK_FLAGS_EXTRA) -D BUILD_SHARED_LIBS=ON +test-shared: build-shared + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target $(RAPIDYAML_NAME)-test-run +test-shared-timing: build-shared + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --target $(RAPIDYAML_NAME)-test-run-timing +$(RAPIDYAML_SO): cfg-shared $(RAPIDYAML_DEPS) + $(CMK_ENV) $(CMAKE) --build $(BDIR)-shared --verbose --parallel --target $(RAPIDYAML_NAME) + cp -fv $(BDIR)-shared/*.so $@ + ln -fs $@ $(RAPIDYAML_LIBNAME).$(SO) + +$(RAPIDYAML_JNI_H): $(JAVAC) $(RAPIDYAML_JAVA) + $(JAVAC) -h . $(RAPIDYAML_JAVA) # $^ doesn't work diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp new file mode 100644 index 000000000..47eeb2b0a --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp @@ -0,0 +1,209 @@ +#include +#include "ysparse_evt.hpp" + +#ifdef __cplusplus +extern "C" { +#endif + + +static C4_NO_INLINE void throw_runtime_exception(JNIEnv * env, const char* msg); +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg); + + +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set(JNIEnv *, jobject, jboolean yes) +{ + ysparse_timing_set(yes); +} + +JNIEXPORT jlong JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1init(JNIEnv *env, jobject) +{ + ysparse *obj = ysparse_init(); + return (jlong)obj; +} + + +JNIEXPORT void JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1destroy(JNIEnv *, jobject, jlong obj) +{ + ysparse_destroy((ysparse*)obj); +} + + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jbyteArray src, jint src_len, + jbyteArray arena, jint arena_len, + jintArray dst, jint dst_len) +{ + TIMED_SECTION("jni:ysparse", (size_type)src_len); + jbyte* arena_ = nullptr; + jbyte* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + jboolean dst_is_copy = false; + jboolean src_is_copy = false; + jboolean arena_is_copy = false; + { + TIMED_SECTION("jni:ysparse/get_jni", (size_type)src_len); + // this is __S__L__O__W__ + // https://stackoverflow.com/questions/43763129/jni-is-getintarrayelements-always-linear-in-time + // https://stackoverflow.com/questions/7395695/how-to-convert-from-bytebuffer-to-integer-and-string + { + TIMED_SECTION("jni:ysparse/GetByteArray(src)"); + src_ = env->GetByteArrayElements(src, &src_is_copy); + } + { + TIMED_SECTION("jni:ysparse/GetByteArray(arena)"); + arena_ = env->GetByteArrayElements(arena, &arena_is_copy); + } + { + TIMED_SECTION("jni:ysparse/GetIntArray(dst)"); + dst_ = env->GetIntArrayElements(dst, &dst_is_copy); + } + { + TIMED_SECTION("jni:ysparse/GetStringUTFChars()"); + filename = env->GetStringUTFChars(jfilename, 0); + } + } + int rc = 0; + { + TIMED_SECTION("jni:ysparse/parse", (size_type)src_len); + try + { + rc = ysparse_parse((ysparse*)obj, filename, + (char*)src_, src_len, + (char*)arena_, arena_len, + dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + { + TIMED_SECTION("jni:ysparse/release"); + // __S__L__O__W__ + { + TIMED_SECTION("jni:ysparse/ReleaseByteArray(src)"); + env->ReleaseByteArrayElements(src, src_, 0); + } + { + TIMED_SECTION("jni:ysparse/ReleaseByteArray(arena)"); + env->ReleaseByteArrayElements(arena, arena_, 0); + } + { + TIMED_SECTION("jni:ysparse/ReleaseIntArray(dst)"); + env->ReleaseIntArrayElements(dst, dst_, 0); + } + { + TIMED_SECTION("jni:ysparse/ReleaseStringUTFChars()"); + env->ReleaseStringUTFChars(jfilename, filename); + } + } + return rc; +} + + +JNIEXPORT jint JNICALL +Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject, + jlong obj, jstring jfilename, + jobject src, jint src_len, + jobject arena, jint arena_len, + jobject dst, jint dst_len) +{ + TIMED_SECTION("jni:ysparse_buf", (size_type)src_len); + char* arena_ = nullptr; + char* src_ = nullptr; + int* dst_ = nullptr; + const char *filename = nullptr; + { + TIMED_SECTION("jni:ysparse_buf/get_jni", (size_type)src_len); + src_ = (char*)env->GetDirectBufferAddress(src); + arena_ = (char*)env->GetDirectBufferAddress(arena); + dst_ = (int*)env->GetDirectBufferAddress(dst); + filename = env->GetStringUTFChars(jfilename, 0); + if(!src_) + throw_runtime_exception(env, "null pointer: src"); + if(!arena_) + throw_runtime_exception(env, "null pointer: arena"); + if(!dst_) + throw_runtime_exception(env, "null pointer: dst"); + } + { + TIMED_SECTION("jni:ysparse_buf/parse", (size_type)src_len); + try + { + return ysparse_parse((ysparse*)obj, filename, + src_, src_len, + arena_, arena_len, + dst_, dst_len); + } + catch (YsParseError const& exc) + { + throw_parse_error(env, exc.location.offset, exc.location.line, exc.location.col, exc.msg.c_str()); + } + catch (std::exception const& exc) + { + throw_runtime_exception(env, exc.what()); + } + } + return 0; // this is executed even if there is an exception +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +static bool s_timing_enabled = false; +RYML_EXPORT bool ysparse_timing_get() +{ + return s_timing_enabled; +} +RYML_EXPORT void ysparse_timing_set(bool yes) +{ + s_timing_enabled = yes; +} + +static C4_NO_INLINE void throw_java_exception(JNIEnv * env, const char* type, const char* msg) +{ + jclass cls = env->FindClass(type); + if (cls != NULL) // if it is null, a NoClassDefFoundError was already thrown + env->ThrowNew(cls, msg); +} + +static C4_NO_INLINE void throw_runtime_exception(JNIEnv *env, const char* msg) +{ + throw_java_exception(env, "java/lang/RuntimeException", msg); +} + +static C4_NO_INLINE void throw_parse_error(JNIEnv *env, size_t offset, size_t line, size_t column, const char *msg) +{ + // see https://stackoverflow.com/questions/55013243/jni-custom-exceptions-with-more-than-one-parameter + jclass cls = env->FindClass("org/rapidyaml/YamlParseErrorException"); + if (cls != NULL) // if it is null, a NoClassDefFoundError was already thrown + { + jstring jmsg = env->NewStringUTF(msg); + jint joffset = (jint)offset; + jint jline = (jint)line; + jint jcol = (jint)column; + // see https://www.rgagnon.com/javadetails/java-0286.html + // about the proper signature. + // we want (int, int, int, String): + const char * const signature = "(IIILjava/lang/String;)V"; + jmethodID ctor = env->GetMethodID(cls, "", signature); + jobject jexc = env->NewObject(cls, ctor, joffset, jline, jcol, jmsg); + env->Throw((jthrowable)jexc); // https://stackoverflow.com/questions/2455668/jni-cast-between-jobect-and-jthrowable + } +} + +#ifdef __cplusplus +} +#endif diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.h b/rapidyaml/native/org_rapidyaml_Rapidyaml.h new file mode 100644 index 000000000..9c84ba2bf --- /dev/null +++ b/rapidyaml/native/org_rapidyaml_Rapidyaml.h @@ -0,0 +1,69 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_rapidyaml_Rapidyaml */ + +#ifndef _Included_org_rapidyaml_Rapidyaml +#define _Included_org_rapidyaml_Rapidyaml +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_init + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1init + (JNIEnv *, jobject); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1destroy + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_timing_set + * Signature: (Z)V + */ +JNIEXPORT void JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1timing_1set + (JNIEnv *, jobject, jboolean); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_parse + * Signature: (JLjava/lang/String;[BI[BI[II)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse + (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint, jbyteArray, jint, jintArray, jint); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_parse_buf + * Signature: (JLjava/lang/String;Ljava/nio/ByteBuffer;ILjava/nio/ByteBuffer;ILjava/nio/IntBuffer;I)Z + */ +JNIEXPORT jboolean JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf + (JNIEnv *, jobject, jlong, jstring, jobject, jint, jobject, jint, jobject, jint); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_reqsize_evt + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1reqsize_1evt + (JNIEnv *, jobject, jlong); + +/* + * Class: org_rapidyaml_Rapidyaml + * Method: ysparse_reqsize_arena + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_rapidyaml_Rapidyaml_ysparse_1reqsize_1arena + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp new file mode 100644 index 000000000..17f7bb7de --- /dev/null +++ b/rapidyaml/native/ysparse_common.hpp @@ -0,0 +1,71 @@ +#pragma once +#ifndef YSPARSE_COMMON_HPP_ +#define YSPARSE_COMMON_HPP_ + +#include +#include + +namespace ryml { +using namespace c4; +using namespace c4::yml; +} // namespace ryml + +using size_type = int; + +struct YsParseError : public std::exception +{ + ryml::Location location; + std::string msg; + const char* what() const noexcept override { return msg.c_str(); } +}; + + +//----------------------------------------------------------------------------- +// timing + +#ifdef __cplusplus +extern "C" { +#endif +RYML_EXPORT bool ysparse_timing_get(); +RYML_EXPORT void ysparse_timing_set(bool yes); +#ifdef __cplusplus +} +#endif + +#ifndef YSPARSE_TIMED +#define TIMED_SECTION(...) +#else +#include +#include +#define TIMED_SECTION(...) timed_section C4_XCAT(ts, __LINE__)(__VA_ARGS__) +struct timed_section +{ + using myclock = std::chrono::steady_clock; + const char* name; + size_type len; + myclock::time_point start; + C4_NO_INLINE timed_section(const char* n, size_type len_=0) + { + if(ysparse_timing_get()) + { + name = n; + len = len_; + start = myclock::now(); + //fprintf(stderr, "%10s : %s...\n", " ", name); + } + } + C4_NO_INLINE ~timed_section() + { + if(ysparse_timing_get()) + { + const std::chrono::duration t = myclock::now() - start; + fprintf(stderr, "%10.6fms: %s", t.count(), name); + if(len) + fprintf(stderr, " %.3fMB/s", (float)len / t.count() * 1.e-3); + fprintf(stderr, "\n"); + } + } +}; +#endif // YSPARSE_TIMED + +#endif // YSPARSE_COMMON_HPP_ diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp new file mode 100644 index 000000000..bce48e16e --- /dev/null +++ b/rapidyaml/native/ysparse_evt.cpp @@ -0,0 +1,74 @@ +#include "ysparse_evt.hpp" + +using namespace ryml; + +#if defined(__cplusplus) +extern "C" { +#endif +// see +// https://stackoverflow.com/questions/230689/best-way-to-throw-exceptions-in-jni-code +// https://stackoverflow.com/questions/4138168/what-happens-when-i-throw-a-c-exception-from-a-native-java-method + +namespace { +C4_NORETURN void ysparse_error(const char* msg, size_t msg_len, Location location, void *user_data) +{ + YsParseError exc; + exc.location = location; + exc.msg.assign(msg, msg_len); + throw exc; +} +} // anon namespace + +RYML_EXPORT ysparse *ysparse_init() +{ + TIMED_SECTION("cpp:ysparse_init"); + Callbacks cb = {}; + cb.m_error = &ysparse_error; + set_callbacks(cb); + ysparse *ryml2evt = _RYML_CB_ALLOC(get_callbacks(), ysparse, 1); + _RYML_CB_CHECK(get_callbacks(), ryml2evt != nullptr); + new ((void*)ryml2evt) ysparse(); + return ryml2evt; +} + +RYML_EXPORT void ysparse_destroy(ysparse *obj) +{ + TIMED_SECTION("cpp:ysparse_destroy"); + obj->~ysparse(); + _RYML_CB_FREE(get_callbacks(), obj, ysparse, 1); +} + +RYML_EXPORT bool ysparse_parse(ysparse *obj, + const char *filename, + char *ys, size_type ys_size, + char *arena, size_type arena_size, + int *events, size_type evt_size) +{ + TIMED_SECTION("cpp:ysparse", ys_size); + csubstr filename_ = filename ? to_csubstr(filename) : csubstr{}; + substr ys_(ys, (size_t)ys_size); + substr arena_(arena, (size_t)arena_size); + { + TIMED_SECTION("cpp:ysparse/reset"); + obj->reset(ys_, arena_, events, evt_size); + } + { + TIMED_SECTION("cpp:ysparse/parse", ys_size); + obj->m_parser.parse_in_place_ev(filename_, ys_); + } + return obj->m_handler.fits_buffers(); +} + +RYML_EXPORT int ysparse_reqsize_evt(ysparse *obj) +{ + return obj->m_handler.required_size_events(); +} + +RYML_EXPORT int ysparse_reqsize_arena(ysparse *obj) +{ + return (int)obj->m_handler.required_size_arena(); +} + +#if defined(__cplusplus) +} +#endif diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp new file mode 100644 index 000000000..dcb1fc490 --- /dev/null +++ b/rapidyaml/native/ysparse_evt.hpp @@ -0,0 +1,105 @@ +#pragma once +#ifndef YSPARSE_EVT_HPP_ +#define YSPARSE_EVT_HPP_ + +#include "c4/yml/parse_engine.hpp" +#include "c4/yml/parse_engine.def.hpp" +#include "c4/yml/extra/event_handler_ints.hpp" +#include "ysparse_common.hpp" + +#if defined(__cplusplus) +extern "C" { +#endif + +struct RYML_EXPORT ysparse +{ + c4::yml::extra::EventHandlerInts m_handler; + c4::yml::ParseEngine m_parser; + ysparse() + : m_handler() + , m_parser(&m_handler) + { + RYML_CHECK(m_parser.options().scalar_filtering()); + } + void reset(c4::substr src, c4::substr arena, int32_t *evt, int32_t evt_size) + { + m_handler.reset(src, arena, evt, evt_size); + } +}; + + +//----------------------------------------------------------------------------- + +/** Initialize the resources */ +RYML_EXPORT ysparse *ysparse_init(); + +/** Destroy the resources */ +RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt); + +/** Parse YAML in the string `ys` of size `ys_size`, and write the + * result into the array of (integer) events `evt` of size + * `evt_size`. Each event is encoded as a mask of evt::EventFlags + * (note that it uses the integer evt::DataType as the underlying + * type), and when an event has an associated string, it is followed + * in the array by two extra values, which encode the offset and the + * length of the string in the `ys` string. The `ys` string is mutated + * during parsing. + * + * @return true if the `evt` and `arena` buffers were large enough to + * accomodate the result The caller must check this value. When false, + * it means that at least one of the buffers could not accomodate the + * result. The caller must then (1) resize `evt` to at least + * the return value, (2) re-copy the original YS into `ys` and (3) + * call again this function, passing in the resized `evt` and the + * fresh copy in `ys`. + * + * @note nothing is written beyond `evt_size` or `arena_size`. This + * means that when `evt_size`/`arena_size` is 0, then `evt`/`arena` + * can be null. This function can be safely called for any valid pair + * of `evt`+`evt_size` and `arena`/`arena_size`, and the same required + * size will always be reported. The same applies for + * `arena`+`arena_size`. + * + * For example, the YAML `say: 2 + 2` produces the following sequence of + * 12 integers: + * + * ```c++ + * BSTR, + * BDOC, + * VAL|BMAP|BLCK, + * KEY|SCLR|PLAI, 0, 3, // "say" + * VAL|SCLR|PLAI, 5, 5, // "2 + 2" + * EMAP, + * EDOC, + * ESTR, + * ``` + * + * Note that the scalar events, ie "say" and "2 + 2", are followed + * each by two extra integers encoding the offset and length of the + * scalar's string. These two extra integers are present whenever the + * event has any of the bits `SCLR`, `ALIA`, `ANCH` or `TAG`. For ease + * of use, there is a bitmask `HAS_STR`, which enables quick testing + * by a simple `flags & HAS_STR`. Refer to evt::EventFlags for the + * full list of flags and their meaning. + * + * Also, where a string requires filtering, the parser filters it + * in-place in the input string, and the extra integers will pertain + * to the resulting filtered string. + */ +RYML_EXPORT bool ysparse_parse(ysparse *ryml2evt, + const char *filename, + char *ys, size_type ys_size, + char *arena, size_type arena_size, + int *evt, size_type evt_size); + +/** Get the required size for the event buffer, from the last parse call */ +RYML_EXPORT int ysparse_reqsize_evt(ysparse *ryml2evt); + +/** Get the required size for the arena buffer, from the last parse call */ +RYML_EXPORT int ysparse_reqsize_arena(ysparse *ryml2evt); + +#if defined(__cplusplus) +} +#endif + +#endif /* YSPARSE_EVT_HPP_ */ diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp new file mode 100644 index 000000000..8d1f49713 --- /dev/null +++ b/rapidyaml/native/ysparse_test.cpp @@ -0,0 +1,728 @@ +#include +#include +#include +#include +#include + +using c4::csubstr; +using c4::substr; +using ryml::Location; + +namespace ievt = c4::yml::extra::ievt; + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +struct Ys2EvtScoped +{ + ysparse *ryml2evt; + Ys2EvtScoped() : ryml2evt(ysparse_init()) {} + ~Ys2EvtScoped() { if(ryml2evt) ysparse_destroy(ryml2evt); } +}; + + +static bool showcmp = false; +struct TestResult +{ + uint32_t num_assertions; + uint32_t num_tests; + uint32_t num_failed_assertions; + uint32_t num_failed_tests; + operator bool() const { return num_failed_tests == 0 && num_failed_assertions == 0; } + void add(TestResult const& that) + { + num_tests += 1 + that.num_tests; + num_assertions += that.num_assertions; + num_failed_tests += (that.num_failed_assertions > 0) + that.num_failed_tests; + num_failed_assertions += that.num_failed_assertions; + } +}; + +// provide a structured input for the events, grouping the relevant +// data in a single structure +struct EvtWithScalar +{ + int flags, str_start, str_len; + csubstr scalar; + bool needs_filter; + EvtWithScalar(int t, int start=0, int len=0, csubstr sclr={}, bool needs_filter_=false) + { + flags = t; + str_start = start; + str_len = len; + scalar = sclr; + needs_filter = needs_filter_; + } + size_t required_size() const { return (flags & ievt::WSTR) ? 3u : 1u; } +}; + +size_t expected_size(std::vector const& evt) +{ + size_t exp = 0; + for(EvtWithScalar const& e : evt) + exp += e.required_size(); + return exp; +} + +#define _runtest(name, ...) \ + do { \ + printf("[ RUN ] %s ... \n", #name); \ + TestResult tr_ = name(__VA_ARGS__); \ + tr.add(tr_); \ + printf("[ %s ] %s\n", tr_?"OK ":"FAIL", #name); \ + } while(0) + +#define CHECK(cond) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s\n", __FILE__, __LINE__, #cond); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + +#define CHECK_EQ(lhs, rhs) \ + do { \ + bool pass = !!(lhs == rhs); \ + ++tr.num_assertions; \ + if(!pass) { \ + std::string slhs = c4::catrs(lhs); \ + std::string srhs = c4::catrs(rhs); \ + printf("%s:%d: fail! %s=%s == %s=%s\n", __FILE__, __LINE__, #lhs, slhs.c_str(), #rhs, srhs.c_str()); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + +#define CHECK_MSG(cond, fmt, ...) \ + do { \ + bool pass = !!(cond); \ + ++tr.num_assertions; \ + if(!pass) { \ + printf("%s:%d: fail! %s:" fmt "\n", __FILE__, __LINE__, #cond, ## __VA_ARGS__); \ + ++tr.num_failed_assertions; \ + } \ + } while(0) + + +struct TestCase +{ + int line; + csubstr ys; + std::vector evt; + +public: + + TestResult test(ysparse *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_evt_large_enough, ); + _runtest(test_evt_too_small, ); + _runtest(test_evt_nullptr, ); + _runtest(test_evt_large_enough_reuse, ryml2evt); + _runtest(test_evt_too_small_reuse, ryml2evt); + _runtest(test_evt_nullptr_reuse, ryml2evt); + return tr; + } + + // happy path: large-enough destination string + TestResult test_evt_large_enough_reuse(ysparse *ryml2evt) const + { + if(evt.empty()) return {}; + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + std::string arena_; + arena_.resize((ys.size() * size_t(3)) / size_t(2)); + substr input = c4::to_substr(input_); + substr arena = c4::to_substr(arena_); + std::vector output; + output.resize(2 * expected_size(evt)); + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + &output[0], (size_type)output.size()); + CHECK(fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + CHECK_MSG((size_t)reqsize_evt == expected_size(evt), "%d vs %zu", reqsize_evt, expected_size(evt)); + CHECK(reqsize_evt != 0); + CHECK(reqsize_evt <= estimated_size); + output.resize(reqsize_evt); + CHECK(testeq(output, input, arena)); + return tr; + } + TestResult test_evt_large_enough() const + { + Ys2EvtScoped lib; + return test_evt_large_enough_reuse(lib.ryml2evt); + } + + // less-happy path: destination string not large enough + TestResult test_evt_too_small_reuse(ysparse *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + std::string arena_; + arena_.resize(0); + substr input = c4::to_substr(input_); + substr arena = c4::to_substr(arena_); + std::vector output; + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + output.data(), (size_type)output.size()); + CHECK(!fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt == expected_size(evt)); + CHECK(reqsize_evt != 0); + CHECK(reqsize_evt <= estimated_size); + output.resize(reqsize_evt); + arena_.resize(reqsize_arena); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + arena = c4::to_substr(arena_); + bool fits_buffers2 = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + output.data(), (size_type)output.size()); + CHECK(fits_buffers2); + int reqsize_evt2 = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena2 = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt2 == reqsize_evt); + CHECK(reqsize_arena2 == reqsize_arena); + output.resize(reqsize_evt2); + CHECK(testeq(output, input, arena)); + return tr; + } + TestResult test_evt_too_small() const + { + Ys2EvtScoped lib; + return test_evt_too_small_reuse(lib.ryml2evt); + } + + // safe calling with nullptr + TestResult test_evt_nullptr_reuse(ysparse *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + int estimated_size = c4::yml::extra::estimate_events_ints_size(input); + bool fits_buffers = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0, + nullptr, 0); + CHECK(!fits_buffers); + int reqsize_evt = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt == expected_size(evt)); + CHECK(reqsize_evt <= estimated_size); + CHECK(reqsize_evt != 0); + std::string arena_; + arena_.resize(reqsize_arena); + substr arena = c4::to_substr(arena_); + std::vector output; + output.resize(reqsize_evt); + input_.assign(ys.begin(), ys.end()); // FIXME + input = c4::to_substr(input_); + bool fits_buffers2 = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + arena.str, (size_type)arena.len, + output.data(), (size_type)output.size()); + CHECK(fits_buffers2); + int reqsize_evt2 = ysparse_reqsize_evt(ryml2evt); + int reqsize_arena2 = ysparse_reqsize_arena(ryml2evt); + CHECK(reqsize_evt2 == reqsize_evt); + CHECK(reqsize_arena2 == reqsize_arena); + CHECK(reqsize_evt2 == output.size()); + CHECK(testeq(output, input, arena)); + return tr; + } + TestResult test_evt_nullptr() const + { + Ys2EvtScoped lib; + return test_evt_nullptr_reuse(lib.ryml2evt); + } + +public: + + bool testeq(std::vector const& actual, csubstr parsed_source, csubstr arena) const + { + int status = true; + size_t num_events_expected = evt.size(); + size_t num_ints_expected = expected_size(evt); + bool same_size = true; + if(actual.size() != num_ints_expected) + { + printf("------\n" + "FAIL: different size\n" + "input:~~~%.*s~~~\n" + "expected size:~~~%zu~~~\n" + "actual size:~~~%zu~~~\n", + (int)ys.len, ys.str, + num_ints_expected, + actual.size()); + same_size = false; + } + for(size_t ia = 0, ie = 0; ie < num_events_expected; ++ie) + { + if(ia >= actual.size()) + { + printf("fail: bad actual size. i=%zu vs %zu=actual.size()=\n", ia, actual.size()); + status = false; + break; + } + #define _testcmp(fmt, cmp, ...) \ + if(showcmp) { printf("status=%d cmp=%d ie=%zu ia=%zu: " fmt "\n", status, (cmp), ie, ia, ## __VA_ARGS__); } \ + status &= (cmp) + char actualbuf_[100]; + char expectedbuf_[100]; + csubstr actualbuf = c4::yml::extra::ievt::to_chars_sub(actualbuf_, actual[ia] & ievt::MASK); + csubstr expectedbuf = c4::yml::extra::ievt::to_chars_sub(expectedbuf_, evt[ie].flags & ievt::MASK); + _testcmp("exp=%d(%.*s) vs act=%d(%.*s)", evt[ie].flags == actual[ia], + evt[ie].flags, (int)expectedbuf.len, expectedbuf.str, + actual[ia], (int)actualbuf.len, actualbuf.str); + status &= (evt[ie].flags == actual[ia]); + if((evt[ie].flags & ievt::WSTR) && (actual[ia] & ievt::WSTR)) + { + csubstr region = (actual[ia] & ievt::AREN) ? arena : parsed_source; + _testcmp(" exp=%d vs act=%d", evt[ie].str_start == actual[ia + 1], evt[ie].str_start, actual[ia + 1]); + _testcmp(" exp=%d vs act=%d", evt[ie].str_len == actual[ia + 2], evt[ie].str_len, actual[ia + 2]); + bool safeactual = (ia + 2 < actual.size()) && (actual[ia + 1] < (int)region.len && actual[ia + 1] + actual[ia + 2] <= (int)region.len); + bool safeexpected = (evt[ie].str_start < (int)region.len && evt[ie].str_start + evt[ie].str_len <= (int)region.len); + _testcmp(" safeactual=%d", safeactual, safeactual); + _testcmp(" safeactual=%d safeexpected=%d", safeactual == safeexpected, safeactual, safeexpected); + if(safeactual && safeexpected) + { + csubstr evtstr = region.sub((size_t)evt[ie].str_start, (size_t)evt[ie].str_len); + csubstr actualstr = region.sub((size_t)actual[ia + 1], (size_t)actual[ia + 2]); + _testcmp(" ref=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evt[ie].scalar == actualstr, + evt[ie].scalar.len, (int)evt[ie].scalar.len, evt[ie].scalar.str, + actualstr.len, (int)actualstr.len, actualstr.str); + if( ! evt[ie].needs_filter) + { + _testcmp(" exp=[%zu]~~~%.*s~~~ vs act=[%zu]~~~%.*s~~~", + evtstr == actualstr, + evtstr.len, (int)evtstr.len, evtstr.str, + actualstr.len, (int)actualstr.len, actualstr.str); + } + } + } + ia += (actual[ia] & ievt::WSTR) ? 3 : 1; + } + if(!status) + printf("------\n" + "FAIL:\n" + "input:~~~%.*s~~~\n", + (int)ys.len, ys.str); + return status && same_size; + } +}; + + +//----------------------------------------------------------------------------- + +struct TestCaseErr +{ + int line; + csubstr ys; + bool is_parse_err; + ryml::Location loc; + + TestCaseErr(int line_, csubstr ys_) : line(line_), ys(ys_), is_parse_err(false), loc() {} + TestCaseErr(int line_, csubstr ys_, ryml::Location loc_) : line(line_), ys(ys_), is_parse_err(true), loc(loc_) {} + + TestResult test(ysparse *ryml2evt) const + { + TestResult tr = {}; + _runtest(test_err, ); + _runtest(test_err_reuse, ryml2evt); + return tr; + } + + TestResult test_err_reuse(ysparse *ryml2evt) const + { + TestResult tr = {}; + std::string input_(ys.begin(), ys.end()); + substr input = c4::to_substr(input_); + bool gotit = false; + try + { + size_type reqsize = ysparse_parse(ryml2evt, "ysfilename", + input.str, (size_type)input.len, + nullptr, 0, + nullptr, 0); + } + catch(YsParseError const& exc) + { + if(is_parse_err) + { + gotit = true; + CHECK_EQ(exc.location.name, "ysfilename"); + CHECK_EQ(exc.location.line, loc.line); + CHECK_EQ(exc.location.col, loc.col); + CHECK_EQ(exc.location.offset, loc.offset); + } + } + catch(std::exception const& exc) + { + if(!is_parse_err) + gotit = true; + } + CHECK(gotit); + return tr; + return tr; + } + TestResult test_err() const + { + Ys2EvtScoped lib; + return test_err_reuse(lib.ryml2evt); + } +}; + + +//----------------------------------------------------------------------------- + +namespace { +// make the declarations shorter +#define tc(ys, ...) {__LINE__, ys, std::vector(__VA_ARGS__)} +#define e(...) EvtWithScalar{__VA_ARGS__} +using namespace ievt; +inline constexpr bool needs_filter = true; +const TestCase test_cases[] = { + // case ------------------------------------------------- + tc("!yamlscript/v0/bare\n--- !code\n42\n", + { + e(BSTR), + e(BDOC), + e(VAL_|TAG_, 0, 19, "!yamlscript/v0/bare"), + e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""), + e(EDOC|PSTR), + e(BDOC|EXPL), + e(VAL_|TAG_, 24, 5, "!code"), + e(VAL_|SCLR|PLAI|PSTR, 30, 2, "42"), + e(EDOC|PSTR), + e(ESTR), + }), + // case ------------------------------------------------- + tc("a: 1", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 3, 1, "1"), + e(EMAP|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("say: 2 + 2", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 3, "say"), + e(VAL_|SCLR|PLAI|PSTR, 5, 5, "2 + 2"), + e(EMAP|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("𝄞: ✅", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 4, "𝄞"), + e(VAL_|SCLR|PLAI|PSTR, 6, 3, "✅"), + e(EMAP|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("[a, b, c]", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"), + e(VAL_|SCLR|PLAI|PSTR, 7, 1, "c"), + e(ESEQ|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc("[a: b]", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|FLOW), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 1, 1, "a"), + e(VAL_|SCLR|PLAI|PSTR, 4, 1, "b"), + e(EMAP|PSTR), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------ + tc(R"(--- !yamlscript/v0 +foo: ! +- {x: y} +- [x, y] +- foo +- 'foo' +- "foo" +- | + foo +- > + foo +- [1, 2, true, false, null] +- &anchor-1 !tag-1 foobar +--- +another: doc +)", + { + e(BSTR), + e(BDOC|EXPL), + e(VAL_|TAG_, 4, 14, "!yamlscript/v0"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 19, 3, "foo"), + e(VAL_|TAG_|PSTR, 24, 1, "!"), + e(VAL_|BSEQ|BLCK|PSTR), + e(VAL_|BMAP|FLOW), + e(KEY_|SCLR|PLAI, 29, 1, "x"), + e(VAL_|SCLR|PLAI|PSTR, 32, 1, "y"), + e(EMAP|PSTR), + e(VAL_|BSEQ|FLOW), + e(VAL_|SCLR|PLAI, 38, 1, "x"), + e(VAL_|SCLR|PLAI|PSTR, 41, 1, "y"), + e(ESEQ|PSTR), + e(VAL_|SCLR|PLAI, 46, 3, "foo"), + e(VAL_|SCLR|SQUO|PSTR, 53, 3, "foo"), + e(VAL_|SCLR|DQUO|PSTR, 61, 3, "foo"), + e(VAL_|SCLR|LITL|PSTR, 70, 4, "foo\n", needs_filter), + e(VAL_|SCLR|FOLD|PSTR, 80, 4, "foo\n", needs_filter), + e(VAL_|BSEQ|FLOW|PSTR), + e(VAL_|SCLR|PLAI, 89, 1, "1"), + e(VAL_|SCLR|PLAI|PSTR, 92, 1, "2"), + e(VAL_|SCLR|PLAI|PSTR, 95, 4, "true"), + e(VAL_|SCLR|PLAI|PSTR, 101, 5, "false"), + e(VAL_|SCLR|PLAI|PSTR, 108, 4, "null"), + e(ESEQ|PSTR), + e(VAL_|TAG_, 126, 6, "!tag-1"), + e(VAL_|ANCH|PSTR, 117, 8, "anchor-1"), + e(VAL_|SCLR|PLAI|PSTR, 133, 6, "foobar"), + e(ESEQ|PSTR), + e(EMAP), + e(EDOC), + e(BDOC|EXPL), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 144, 7, "another"), + e(VAL_|SCLR|PLAI|PSTR, 153, 3, "doc"), + e(EMAP|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"(plain: well + a + b + c +squo: 'single''quote' +dquo: "x\t\ny" +lit: | + X + Y + Z +fold: > + U + V + W +)", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 5, "plain"), + e(VAL_|SCLR|PLAI|PSTR, 7, 10, "well a b c"), + e(KEY_|SCLR|PLAI|PSTR, 24, 4, "squo"), + e(VAL_|SCLR|SQUO|PSTR, 31, 12, "single'quote", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 46, 4, "dquo"), + e(VAL_|SCLR|DQUO|PSTR, 53, 4, "x\t\ny", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 61, 3, "lit"), + e(VAL_|SCLR|LITL|PSTR, 68, 6, "X\nY\nZ\n", needs_filter), + e(KEY_|SCLR|PLAI|PSTR, 89, 4, "fold"), + e(VAL_|SCLR|FOLD|PSTR, 97, 6, "U V W\n", needs_filter), + e(EMAP|PSTR), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc("- !!seq []", + { + e(BSTR), + e(BDOC), + e(VAL_|BSEQ|BLCK), + e(VAL_|TAG_, 2, 5, "!!seq"), + e(VAL_|BSEQ|FLOW|PSTR), + e(ESEQ), + e(ESEQ), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(defn run(prompt session=nil): + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 0, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 32, 12, "when session"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 50, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL|PSTR, 83, 54, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n", needs_filter), + e(EMAP|PSTR), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), + // case ------------------------------------------------- + tc(R"_(#!/usr/bin/env ys-0 + +defn run(prompt session=nil): + session-text =: + when session && session:fs-e: + + answer =: + cond: + api-model =~ /^dall-e/: + openai-image(prompt).data.0.url + api-model.in?(anthropic-models): + anthropic(prompt):anthropic-message:format + api-model.in?(groq-models): + groq(prompt).choices.0.message.content:format + api-model.in?(openai-models): + openai-chat(prompt).choices.0.message.content:format + else: die() + + say: answer + + when session: + write session _ :append true: |+ + Q: $(orig-prompt:trim) + A ($api-model): + $(answer:trim) + +)_", + { + e(BSTR), + e(BDOC), + e(VAL_|BMAP|BLCK), + e(KEY_|SCLR|PLAI, 21, 28, "defn run(prompt session=nil)"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 53, 14, "session-text ="), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 73, 28, "when session && session:fs-e"), + e(VAL_|SCLR|PLAI|PSTR, 0, 0, ""), // note empty scalar pointing at the front + e(EMAP|PSTR), + e(KEY_|SCLR|PLAI, 106, 8, "answer ="), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 120, 4, "cond"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 132, 22, "api-model =~ /^dall-e/"), + e(VAL_|SCLR|PLAI|PSTR, 164, 31, "openai-image(prompt).data.0.url"), + e(KEY_|SCLR|PLAI|PSTR, 202, 31, "api-model.in?(anthropic-models)"), + e(VAL_|SCLR|PLAI|PSTR, 243, 42, "anthropic(prompt):anthropic-message:format"), + e(KEY_|SCLR|PLAI|PSTR, 292, 26, "api-model.in?(groq-models)"), + e(VAL_|SCLR|PLAI|PSTR, 328, 45, "groq(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI|PSTR, 380, 28, "api-model.in?(openai-models)"), + e(VAL_|SCLR|PLAI|PSTR, 418, 52, "openai-chat(prompt).choices.0.message.content:format"), + e(KEY_|SCLR|PLAI|PSTR, 477, 4, "else"), + e(VAL_|SCLR|PLAI|PSTR, 483, 5, "die()"), + e(EMAP|PSTR), + e(EMAP), + e(KEY_|SCLR|PLAI, 492, 3, "say"), + e(VAL_|SCLR|PLAI|PSTR, 497, 6, "answer"), + e(KEY_|SCLR|PLAI|PSTR, 507, 12, "when session"), + e(VAL_|BMAP|BLCK|PSTR), + e(KEY_|SCLR|PLAI, 525, 28, "write session _ :append true"), + e(VAL_|SCLR|LITL|PSTR, 558, 55, "Q: $(orig-prompt:trim)\nA ($api-model):\n$(answer:trim)\n\n", needs_filter), + e(EMAP|PSTR), + e(EMAP), + e(EMAP), + e(EDOC), + e(ESTR), + }), +}; +#define tcf(...) TestCaseErr(__LINE__, __VA_ARGS__) +const TestCaseErr test_cases_err[] = { + tcf("- !!str, xxx\n", Location(13, 2, 1)), + //FIXME tcf(": : : :", Location(2, 1, 3)), +}; +} // namespace + + +//----------------------------------------------------------------------------- + +int main(int argc, const char *argv[]) +{ + for(int i = 1; i < argc; ++i) + { + csubstr arg = ryml::to_csubstr(argv[i]); + if(arg == "--timing" || arg == "-t") + ysparse_timing_set(true); + } + Ys2EvtScoped ys2evt; + TestResult total = {}; + size_t failed_cases = {}; + size_t num_cases = C4_COUNTOF(test_cases); + for(size_t i = 0; i < C4_COUNTOF(test_cases); ++i) + { + printf("-----------------------------------------\n" + "%s:%d: case %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", + __FILE__, test_cases[i].line, + i, num_cases, test_cases[i].ys.len, (int)test_cases[i].ys.len, test_cases[i].ys.str); + const TestResult tr = test_cases[i].test(ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("%s:%d: case %zu/%zu: %s\n", + __FILE__, test_cases[i].line, + i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + size_t num_cases_err = C4_COUNTOF(test_cases_err); + for(size_t i = 0; i < C4_COUNTOF(test_cases_err); ++i) + { + printf("-----------------------------------------\n" + "%s:%d: errcase %zu/%zu ...\n" + "[%zu]~~~%.*s~~~\n", + __FILE__, test_cases[i].line, + i, num_cases_err, test_cases_err[i].ys.len, (int)test_cases_err[i].ys.len, test_cases_err[i].ys.str); + const TestResult tr = test_cases_err[i].test(ys2evt.ryml2evt); + total.add(tr); + failed_cases += (!tr); + printf("%s:%d: case %zu/%zu: %s\n", + __FILE__, test_cases[i].line, + i, C4_COUNTOF(test_cases), tr ? "ok!" : "failed"); + } + printf("assertions: %u/%u pass %u/%u fail\n", total.num_assertions - total.num_failed_assertions, total.num_assertions, total.num_failed_assertions, total.num_assertions); + printf("tests: %u/%u pass %u/%u fail\n", total.num_tests - total.num_failed_tests, total.num_tests, total.num_failed_tests, total.num_tests); + printf("cases: %zu/%zu pass %zu/%zu fail\n", num_cases-failed_cases, num_cases+num_cases_err, failed_cases, num_cases); + if(total) + printf("TESTS SUCCEED!\n"); + return total ? 0 : -1; +} diff --git a/rapidyaml/pom.xml b/rapidyaml/pom.xml new file mode 100644 index 000000000..54902b7fd --- /dev/null +++ b/rapidyaml/pom.xml @@ -0,0 +1,115 @@ + + + + + rapidyaml + + 0.8.0 + + rapidyaml + + org.rapidyaml + + 4.0.0 + + + rapidyaml is a C++ library to parse and emit YAML, and do it fast. + + + https://rapidyaml.org + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + + clojars + Clojars repository + https://clojars.org/repo + + + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 3.8.1 + + + org.json + json + 20240205 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + + + + + + maven-project-info-reports-plugin + + + + diff --git a/rapidyaml/src/README.md b/rapidyaml/src/README.md new file mode 100644 index 000000000..d920cfa6b --- /dev/null +++ b/rapidyaml/src/README.md @@ -0,0 +1,33 @@ +## Notes on JNI vs JNA +, packaging, loading, etc + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > JNI is almost always the fasted ffi option. I've heard good things about JNR performance. JNA is usually one of the slower ffi options, but can be sped up with direct mapping, https://github.com/java-native-access/jna/blob/master/www/DirectMapping.md. + + +## Notes on how to profile + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > Another tool to try to find the bottleneck is https://github.com/clojure-goes-fast/clj-async-profiler. The flamegraph might show some obvious performance issue (assuming the issue is on the jvm side). + + > Depending on how long a parse takes, I would recommend something like https://github.com/hugoduncan/criterium. time is not a good way to benchmark code unless it's a very slow function call. I would use the profiler to try and figure out where the bottleneck is. + + +## JNI examples + +- [full JNI example](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo031) +- [full JNI example with other non-JNI shared libraries](https://github.com/mkowsiak/jnicookbook/tree/master/recipes/recipeNo035) +- [another example linking with more libraries](https://www.dynamsoft.com/codepool/package-jni-shared-library-jar-file.html) + +- https://stackoverflow.com/questions/1611357/how-to-make-a-jar-file-that-includes-dll-files#comment1483970_1611367 + +## Notes on JNI - how to call c++ code from java + +From a [thread on slack](https://app.slack.com/client/T03RZGPFR/activity) + + > I'm surprised trying to pass a mutable byte buffer doesn't cause more issues. I think the recommended way to pass a byte buffer to native is with http://java-native-access.github.io/jna/5.13.0/javadoc/com/sun/jna/Memory.html and you can get the string with .getString. + + > > But I was thinking whether it is possible/practical/advisable (in terms of speed) to build the Clojure dictionary directly in the C++ code. Currently the C++ code is providing an EDN markup string that is later parsed in Clojure. Assuming a large dictionary of say ~40k entries, would it be possible to call native clojure/java JNI functions to build the final structure instead of creating the intermediate EDN? Would that be a gain? Would the ~40k calls to JNI end up costing too much? + > This isn't something I've tried before, so take it with a grain of salt, but you have at least a few options with different tradeoffs. All the collection types in clojure are based on protocols/interfaces, so it would be possible to just return a pointer, with no copying, and wrap it proxy that implements all the relevant interfaces for maps/lists/etc. When JVM code asks for a value from a map or element from list, you produce the JVM value for numbers/strings or you return another proxy pointer if it's a collection. You might still have to make a copy to return a string value. If you're returning large values that you expect will only be partially read or read only once, then lazily producing jvm values might be a win. If you expect the large value to be completely read multiple times, then it could be potentially faster to just convert the full data structure to a JVM value. There's also intermediate options where you do some of the work upfront, and do some of the work lazily. Granularity will also affect memory usage. You probably don't want some scenario where someone parses a giant blob and keeps only a small part, but still has to hold the giant value in memory until the small part gets reclaimed.I don't have a good answer for you here. My intuition is that your approach of building the final data structure in c++ is probably a good idea, but I don't really have the experience to say for sure.This type of question might get a better answer in #data-science. I think they similar issues with dealing with large datasets that are partially processed in native code. They've also built deep integrations with python via https://github.com/clj-python/libpython-clj where I think they've run into similar problems. diff --git a/rapidyaml/src/main/java/org/rapidyaml/Evt.java b/rapidyaml/src/main/java/org/rapidyaml/Evt.java new file mode 100644 index 000000000..0e156efe9 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Evt.java @@ -0,0 +1,71 @@ +package org.rapidyaml; + +public class Evt { + // Event types + public static final int BSTR = 1 << 0; // +STR + public static final int ESTR = 1 << 1; // -STR + public static final int BDOC = 1 << 2; // +DOC + public static final int EDOC = 1 << 3; // -DOC + public static final int BMAP = 1 << 4; // +MAP + public static final int EMAP = 1 << 5; // -MAP + public static final int BSEQ = 1 << 6; // +SEQ + public static final int ESEQ = 1 << 7; // -SEQ + public static final int SCLR = 1 << 8; // =VAL + public static final int ALIA = 1 << 9; // =ALI + + // Style flags + public static final int PLAI = 1 << 10; // : (plain scalar) + public static final int SQUO = 1 << 11; // ' (single-quoted scalar) + public static final int DQUO = 1 << 12; // " (double-quoted scalar) + public static final int LITL = 1 << 13; // | (block literal scalar) + public static final int FOLD = 1 << 14; // > (block folded scalar) + + public static final int FLOW = 1 << 15; // flow container: + // [] for seqs or {} for maps + public static final int BLCK = 1 << 16; // block container + + // Modifiers + public static final int ANCH = 1 << 17; // anchor + public static final int TAG_ = 1 << 18; // tag + + // Structure flags + public static final int KEY_ = 1 << 19; // as key + public static final int VAL_ = 1 << 20; // as value + public static final int EXPL = 1 << 21; // --- (with BDOC) or + // ... (with EDOC) + // (may be fused with FLOW + // if needed) + + // Directives + public static final int YAML = 1 << 22; // `%YAML ` followed by version string + public static final int TAGD = 1 << 23; // tag directive name : `%TAG .......` followed by name string + public static final int TAGV = 1 << 24; // tag directive value: `%TAG ...... ` followed by value string + + // Buffer flags + ///< IMPORTANT. Marks events whose string was placed in the + ///< arena. Fhis happens when the filtered string is larger than the + ///< original string in the YAML code (eg from tags that resolve to + ///< a larger string, or from "\L" or "\P" in double quotes, which + ///< expand from two to three bytes). Because of this size + ///< expansion, the filtered string cannot be placed in the original + ///< source and needs to be placed in the arena. + public static final int AREN = 1 << 25; + ///< special flag to enable look back in the event array. it + ///< signifies that the previous event has a string, meaning that + ///< the jump back to that event is 3 positions. without this flag it + ///< would be impossible to jump to the previous event + public static final int PSTR = 1 << 26; + ///< special flag to mark a scalar as unfiltered (when the parser + ///< is set not to filter) + public static final int UNFILT = 1 << 27; + + // Utility flags/masks + public static final int LAST = UNFILT; ///< the last flag defined above + public static final int MASK = (LAST << 1) - 1; ///< a mask of all bits in this enumeration + /// with string: mask of all the events that encode a string + /// following the event. in the event has a string. the next two + /// integers will provide respectively the string's offset and + /// length. See also @ref PSTR. + public static final int WSTR = SCLR|ALIA|ANCH|TAG_|TAGD|TAGV|YAML; + +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java b/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java new file mode 100644 index 000000000..68b4b35d7 --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java @@ -0,0 +1,59 @@ +package org.rapidyaml; + +import java.io.File; +import java.io.InputStream; +import java.io.IOException; + +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +class NativeLibLoader { + public static void loadLibraryFromResource(String libraryName) + throws IOException + { +// System.out.println("libraryName: " + libraryName); + ClassLoader classLoader = + Thread.currentThread().getContextClassLoader(); + + InputStream inputStream = + classLoader.getResourceAsStream(libraryName); + + if (inputStream == null) + throw new IOException( + "Failed to load library resource '" + + libraryName + "' from NativeLibLoader"); + +// System.out.println("inputStream: " + inputStream); + + File tempDir = createTempDir(); + // tempDir.deleteOnExit(); + File tempFile = new File(tempDir, libraryName); +// System.out.println("tempFile: " + tempFile); + + Files.copy( + inputStream, + tempFile.toPath(), + StandardCopyOption.REPLACE_EXISTING); + + inputStream.close(); + + try { + System.load(tempFile.getAbsolutePath()); + } + finally { + // tempFile.delete(); + } + +// System.out.println("Loaded library from temp file"); + } + + private static File createTempDir() throws IOException { + String tempBase = System.getProperty("java.io.tmpdir"); + File tempDir = new File(tempBase, "" + System.nanoTime()); + if (! tempDir.mkdir()) + throw new IOException( + "Failed to create temp directory " + + tempDir.getName()); + return tempDir; + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java new file mode 100644 index 000000000..c9cd8048d --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java @@ -0,0 +1,169 @@ +package org.rapidyaml; + +import org.rapidyaml.NativeLibLoader; + +import java.io.IOException; + +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ByteOrder; + +/** + * Interface with the librapidyaml shared library + */ +public class Rapidyaml { + public static String RAPIDYAML_NAME = "ysparse.0.9.0"; + public static String RAPIDYAML_LIBNAME = "ysparse.0.9.0.so"; + + private final long ysparse; ///< pointer to the c++ object + private native long ysparse_init(); + private native void ysparse_destroy(long ysparse); + private native void ysparse_timing_set(boolean yes); + private native boolean ysparse_parse(long ysparse, String filename, + byte[] ys, int ys_length, + byte[] arena, int arena_length, + int[] evt, int evt_length); + private native boolean ysparse_parse_buf(long ysparse, String filename, + ByteBuffer ys, int ys_length, + ByteBuffer arena, int arena_length, + IntBuffer evt, int evt_length); + private native int ysparse_reqsize_evt(long ysparse); + private native int ysparse_reqsize_arena(long ysparse); + + + // XXX this 'main' is for testing + public static void main( + String[] args + ) throws Exception, IOException { + (new Rapidyaml()).timingEnabled(true); + System.out.printf("It works!\n"); + } + + + + //------------------------ + // CTOR/DTOR + //------------------------ + + public Rapidyaml() throws Exception, IOException { + if (System.getenv("YS_TESTING") != null) + System.loadLibrary(RAPIDYAML_NAME); + else + NativeLibLoader.loadLibraryFromResource(RAPIDYAML_LIBNAME); + + this.ysparse = this.ysparse_init(); + timingEnabled(false); + } + + // Likely bad idea to implement finalize: + // + // https://stackoverflow.com/questions/158174/why-would-you-ever-implement-finalize + // +// protected void finalize() throws Throwable { +// try { +// this.ysparse_destroy(this.ysparse); +// } +// finally { +// super.finalize(); +// } +// } + + + //------------------------ + // EVT + //------------------------ + + public boolean parseYs(byte[] src, byte[] arena, int[] evts) throws Exception + { + return parseYs("yamlscript", src, arena, evts); + } + + public boolean parseYsDirect(ByteBuffer src, ByteBuffer arena, IntBuffer evt) throws Exception + { + return parseYsDirect("yamlscript", src, arena, evt); + } + + public boolean parseYs(String filename, byte[] src, byte[] arena, int[] evts) throws Exception + { + long t = timingStart("ysparse"); + boolean fits_buffers = ysparse_parse(this.ysparse, filename, + src, src.length, + arena, arena.length, + evts, evts.length); + timingStop("ysparse", t, src.length); + return fits_buffers; + } + + public boolean parseYsDirect(String filename, ByteBuffer src, ByteBuffer arena, IntBuffer evt) throws Exception { + if (! src.isDirect()) + throw new RuntimeException("src must be direct"); + if (! arena.isDirect()) + throw new RuntimeException("arena must be direct"); + if (! evt.isDirect()) + throw new RuntimeException("evt must be direct"); + // the byte order for src+arena does not matter + // but for evt it really does: + if (evt.order() != ByteOrder.nativeOrder()) + throw new RuntimeException("evt byte order must be native"); + long t = timingStart("ysparseBuf"); + evt.position(evt.capacity()); + boolean fits_buffers = ysparse_parse_buf(this.ysparse, filename, + src, src.position(), + arena, arena.position(), + evt, evt.capacity()); + if (fits_buffers) + evt.position(ysparse_reqsize_evt(this.ysparse)); + timingStop("ysparseBuf", t, src.position()); + return fits_buffers; + } + + /** Get the required size for the event output buffer, from the last parse call */ + public int requiredSizeEvt() { return ysparse_reqsize_evt(this.ysparse); } + /** Get the required size for the arena buffer, from the last parse call */ + public int requiredSizeArena() { return ysparse_reqsize_arena(this.ysparse); } + + /** a helper to create a direct int buffer */ + public static IntBuffer mkIntBuffer(int numInts) { + ByteBuffer bb = ByteBuffer.allocateDirect(/*numBytes*/4 * numInts); + // !!! need to explicitly set the byte order to the native order + return bb.order(ByteOrder.nativeOrder()).asIntBuffer(); + } + + //------------------------ + // TIME + //------------------------ + + private boolean showTiming = true; + + public void timingEnabled(boolean yes) { + showTiming = yes; + ysparse_timing_set(yes); + } + + private long timingStart(String name) { + if(showTiming) { + System.out.printf(" java:%s...\n", name); + return System.nanoTime(); + } + return 0; + } + + private void timingStop(String name, long t) { + if(showTiming) { + t = System.nanoTime() - t; + System.out.printf( + " java:%s: %.6fms\n", name, (float) t/1.e6f); + } + } + + private void timingStop(String name, long t, int numBytes) { + if(showTiming) { + t = System.nanoTime() - t; + float dt = (float)t; + float fb = (float)numBytes; + System.out.printf( + " java:%s: %.6fms %.3fMB/s %dB\n", + name, dt/1.e6f, 1.e3f*fb/dt, numBytes); + } + } +} diff --git a/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java new file mode 100644 index 000000000..d5d4730df --- /dev/null +++ b/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java @@ -0,0 +1,16 @@ +package org.rapidyaml; + +// https://www.baeldung.com/java-new-custom-exception +public class YamlParseErrorException extends Exception +{ + public final int offset; + public final int line; + public final int column; + public YamlParseErrorException(int offset_, int line_, int column_, String msg) + { + super(msg); + offset = offset_; + line = line_; + column = column_; + } +} diff --git a/rapidyaml/src/site/site.xml b/rapidyaml/src/site/site.xml new file mode 100644 index 000000000..27de8ef17 --- /dev/null +++ b/rapidyaml/src/site/site.xml @@ -0,0 +1,26 @@ + + + + + rapidyaml + https://maven.apache.org/images/apache-maven-project.png + https://www.apache.org/ + + + + https://maven.apache.org/images/maven-logo-black-on-white.png + https://maven.apache.org/ + + + + org.apache.maven.skins + maven-fluido-skin + 1.7 + + + + + + + diff --git a/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java new file mode 100644 index 000000000..d32bb7bcc --- /dev/null +++ b/rapidyaml/src/test/java/org/rapidyaml/RapidyamlTest.java @@ -0,0 +1,574 @@ +package org.rapidyaml; + +import org.rapidyaml.Rapidyaml; +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.IntBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + + +/** + * Unit test for simple App. + */ +public class RapidyamlTest extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public RapidyamlTest(String testName) + { + super(testName); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite(RapidyamlTest.class); + } + + public void testByteBuffer2Slice2Str() + { + String str = "0123456789𝄞✅★"; + byte[] arr = str.getBytes(StandardCharsets.UTF_8); + ByteBuffer buf = ByteBuffer.allocateDirect(arr.length); + buf.put(arr); + assertEquals("012", StandardCharsets.UTF_8.decode(buf.slice(0, 3)).toString()); + assertEquals("123", StandardCharsets.UTF_8.decode(buf.slice(1, 3)).toString()); + assertEquals("234", StandardCharsets.UTF_8.decode(buf.slice(2, 3)).toString()); + assertEquals("345", StandardCharsets.UTF_8.decode(buf.slice(3, 3)).toString()); + assertEquals("456", StandardCharsets.UTF_8.decode(buf.slice(4, 3)).toString()); + assertEquals("0123", StandardCharsets.UTF_8.decode(buf.slice(0, 4)).toString()); + assertEquals("1234", StandardCharsets.UTF_8.decode(buf.slice(1, 4)).toString()); + assertEquals("2345", StandardCharsets.UTF_8.decode(buf.slice(2, 4)).toString()); + assertEquals("3456", StandardCharsets.UTF_8.decode(buf.slice(3, 4)).toString()); + assertEquals("4567", StandardCharsets.UTF_8.decode(buf.slice(4, 4)).toString()); + assertEquals("01234", StandardCharsets.UTF_8.decode(buf.slice(0, 5)).toString()); + assertEquals("12345", StandardCharsets.UTF_8.decode(buf.slice(1, 5)).toString()); + assertEquals("23456", StandardCharsets.UTF_8.decode(buf.slice(2, 5)).toString()); + assertEquals("34567", StandardCharsets.UTF_8.decode(buf.slice(3, 5)).toString()); + assertEquals("45678", StandardCharsets.UTF_8.decode(buf.slice(4, 5)).toString()); + assertEquals("𝄞", StandardCharsets.UTF_8.decode(buf.slice(10, 4)).toString()); + assertEquals("𝄞✅", StandardCharsets.UTF_8.decode(buf.slice(10, 7)).toString()); + assertEquals("𝄞✅★", StandardCharsets.UTF_8.decode(buf.slice(10, 10)).toString()); + assertEquals("✅", StandardCharsets.UTF_8.decode(buf.slice(14, 3)).toString()); + assertEquals("✅★", StandardCharsets.UTF_8.decode(buf.slice(14, 6)).toString()); + assert("012".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 3)))); + assert("123".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 3)))); + assert("234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 3)))); + assert("345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 3)))); + assert("456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 3)))); + assert("0123".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 4)))); + assert("1234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 4)))); + assert("2345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 4)))); + assert("3456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 4)))); + assert("4567".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 4)))); + assert("01234".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(0, 5)))); + assert("12345".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(1, 5)))); + assert("23456".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(2, 5)))); + assert("34567".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(3, 5)))); + assert("45678".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(4, 5)))); + assert("𝄞".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 4)))); + assert("𝄞✅".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 7)))); + assert("𝄞✅★".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(10, 10)))); + assert("✅".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(14, 3)))); + assert("✅★".contentEquals(StandardCharsets.UTF_8.decode(buf.slice(14, 6)))); + } + + public void testPass() { } + + public void testPlainMap() + { + String ys = "a: 1"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 1, "a"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 3, 1, "1"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testUtf8() + { + String ys = "𝄞: ✅"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "𝄞"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "✅"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testUtf8_2() + { + String ys = "star: ★"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 4, "star"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 6, 3, "★"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testTaggedInt() + { + String ys = "- !!int 42"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!int"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 8, 2, "42"), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testTaggedSeq() + { + String ys = "- !!seq []"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.TAG_, 2, 5, "!!seq"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.ESEQ), + mkev(Evt.ESEQ), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testDocTag() + { + String ys = "!yamlscript/v0/bare\n--- !code\n42\n"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.TAG_, 0, 19, "!yamlscript/v0/bare"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 0, 0, ""), + mkev(Evt.EDOC), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.TAG_, 24, 5, "!code"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 30, 2, "42"), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testLargeCase() + { + String ys = "--- !yamlscript/v0\n" + + "foo: !\n" + + "- {x: y}\n" + + "- [x, y]\n" + + "- foo\n" + + "- 'foo'\n" + + "- \"foo\"\n" + + "- |\n" + + " foo\n" + + " literal\n" + + "- >\n" + + " foo\n" + + " folded\n" + + "- [1, 2, true, false, null]\n" + + "- &anchor-1 !tag-1 foobar\n" + + "---\n" + + "another: doc\n"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.TAG_, 4, 14, "!yamlscript/v0"), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 19, 3, "foo"), + mkev(Evt.VAL_|Evt.TAG_, 25, 0, ""), + mkev(Evt.VAL_|Evt.BSEQ|Evt.BLCK), + mkev(Evt.VAL_|Evt.BMAP|Evt.FLOW), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 29, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 32, 1, "y"), + mkev(Evt.EMAP), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 38, 1, "x"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 41, 1, "y"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 46, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 53, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 61, 3, "foo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 70, 12, "foo\nliteral\n"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 98, 11, "foo folded\n"), + mkev(Evt.VAL_|Evt.BSEQ|Evt.FLOW), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 124, 1, "1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 127, 1, "2"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 130, 4, "true"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 136, 5, "false"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 143, 4, "null"), + mkev(Evt.ESEQ), + mkev(Evt.VAL_|Evt.TAG_, 162, 5, "tag-1"), + mkev(Evt.VAL_|Evt.ANCH, 152, 8, "anchor-1"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 168, 6, "foobar"), + mkev(Evt.ESEQ), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.BDOC|Evt.EXPL), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 179, 7, "another"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 188, 3, "doc"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testFilterCase() + { + String ys = "" + + "plain: well\n" + + " a\n" + + " b\n" + + " c\n" + + "squo: 'single''quote'\n" + + "dquo: \"x\\t\\ny\"\n" + + "lit: |\n" + + " X\n" + + " Y\n" + + " Z\n" + + "fold: >\n" + + " U\n" + + " V\n" + + " W\n"; + ExpectedEvent[] expected = { + mkev(Evt.BSTR), + mkev(Evt.BDOC), + mkev(Evt.VAL_|Evt.BMAP|Evt.BLCK), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 0, 5, "plain"), + mkev(Evt.VAL_|Evt.SCLR|Evt.PLAI, 7, 10, "well a b c"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 24, 4, "squo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.SQUO, 31, 12, "single'quote"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 46, 4, "dquo"), + mkev(Evt.VAL_|Evt.SCLR|Evt.DQUO, 53, 4, "x\t\ny"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 61, 3, "lit"), + mkev(Evt.VAL_|Evt.SCLR|Evt.LITL, 68, 6, "X\nY\nZ\n"), + mkev(Evt.KEY_|Evt.SCLR|Evt.PLAI, 89, 4, "fold"), + mkev(Evt.VAL_|Evt.SCLR|Evt.FOLD, 97, 6, "U V W\n"), + mkev(Evt.EMAP), + mkev(Evt.EDOC), + mkev(Evt.ESTR), + }; + testEvt_(ys, expected); + } + + public void testFailure() throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = "{a: b"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] arena = new byte[src.length]; + byte[] srcbuf = new byte[src.length]; + boolean gotit = false; + try { + callParse(src, srcbuf, arena); + } + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(5, e.offset); + assertEquals(1, e.line); + assertEquals(6, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); + } + catch(RuntimeException e) { + fail("wrong exception type"); + } + catch(Exception e) { + fail("wrong exception type"); + } + assertTrue(gotit); + } + + public void testFailureBuf() throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + String ys = "{a: b"; + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer arena = ByteBuffer.allocateDirect(src.length); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + boolean gotit = false; + try { + callParseDirect(src, bbuf, arena); + } + catch(YamlParseErrorException e) { + gotit = true; + assertEquals(5, e.offset); + assertEquals(1, e.line); + assertEquals(6, e.column); + assertTrue(e.getMessage() != null); + assertFalse(e.getMessage().isEmpty()); + } + catch(RuntimeException e) { + fail("wrong exception type"); + } + catch(Exception e) { + fail("wrong exception type"); + } + assertTrue(gotit); + } + + + private void testEvt_(String ys, ExpectedEvent[] expected) + { + byte[] src = ys.getBytes(StandardCharsets.UTF_8); + byte[] srcbuf = new byte[src.length]; + byte[] arenaarr = new byte[src.length]; + int[] actual; + try { + actual = callParse(src, srcbuf, arenaarr); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evt (no buf)"); + throw e; + } + //------ + src = ys.getBytes(StandardCharsets.UTF_8); + ByteBuffer arenabuf = ByteBuffer.allocateDirect(src.length); + ByteBuffer bbuf = ByteBuffer.allocateDirect(src.length); + bbuf.put(src); + IntBuffer buf; + try { + buf = callParseDirect(src, bbuf, arenabuf); + actual = buf2arr(buf); + } + catch (Exception e) { + fail("parse error:\n" + e.getMessage()); + actual = new int[1]; + } + try { + cmpEvt_(ys, srcbuf, actual, expected); + } + catch (Exception e) { + System.err.printf("error: evtbuf"); + throw e; + } + } + + boolean dbglog = false; + private void cmpEvt_(String ys, byte[] src, int[] actual, ExpectedEvent[] expected) + { + if(dbglog) { + System.out.printf("----------------------\n~~~\n%s\n~~~\n", ys); + } + int numEvts = actual.length; + try { + int ia = 0; + int ie = 0; + int status = 1; + while(true) { + if((ia < numEvts) != (ie < expected.length)) { + System.out.printf("status=%d szActual=%d szExpected=%d\n", status, numEvts, ExpectedEvent.required_size_(expected)); + status = 0; + break; + } + if(ia >= numEvts) + break; + if(ie >= expected.length) + break; + int cmp = 1; + if(dbglog) + System.out.printf("status=%d evt=%d pos=%d expflags=%d actualflags=%d", status, ie, ia, expected[ie].flags, actual[ia]); + cmp &= (expected[ie].flags == actual[ia]) ? 1 : 0; + if(((actual[ia] & Evt.HAS_STR) != 0) && ((expected[ie].flags & Evt.HAS_STR)) != 0) { + cmp &= (ia + 2 < numEvts) ? 1 : 0; + if(cmp != 0) { + cmp &= (expected[ie].str_start == actual[ia + 1]) ? 1 : 0; + cmp &= (expected[ie].str_len == actual[ia + 2]) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=(%d,%d) actual=(%d,%d)", expected[ie].str_start, expected[ie].str_len, actual[ia + 1], actual[ia + 2]); + if(cmp != 0) { + cmp &= (actual[ia + 1] >= 0) ? 1 : 0; + cmp &= (actual[ia + 2] >= 0) ? 1 : 0; + cmp &= (actual[ia + 1] + actual[ia + 2] <= src.length) ? 1 : 0; + if(cmp != 0) { + String actualStr = new String(src, actual[ia + 1], actual[ia + 2], StandardCharsets.UTF_8); + cmp &= actualStr.equals(expected[ie].str) ? 1 : 0; + if(dbglog) + System.out.printf(" exp=~~~%s~~~ actual=~~~%s~~~", expected[ie].str, actualStr); + } + else { + if(dbglog) + System.out.printf(" BAD RANGE len=%d", src.length); + } + } + } + } + if(dbglog) + System.out.printf(" --> %s\n", cmp != 0 ? "ok!" : "FAIL"); + status &= cmp; + ia += ((actual[ia] & Evt.HAS_STR) != 0) ? 3 : 1; + ++ie; + } + if(ExpectedEvent.required_size_(expected) != numEvts) + status = 0; + assertEquals(1, status); + } + catch (Exception e) { + System.err.println("expected:"); + System.err.println(expected); + System.err.println("actual"); + System.err.println(actual); + throw e; + } + } + + public static String buf2str(ByteBuffer edn) + { + int size = edn.position(); + size = size > 0 ? size - 1 : 0; + edn.position(0); + edn.limit(size); + return StandardCharsets.UTF_8.decode(edn).toString(); + } + + public static int[] buf2arr(IntBuffer evt) + { + int[] ret = new int[evt.position()]; + for(int i = 0; i < evt.position(); ++i) { + ret[i] = evt.get(i); + } + return ret; + } + + static int[] callParse(byte[] src, byte[] srcbuf, byte[] arena) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + System.arraycopy(src, 0, srcbuf, 0, src.length); + int[] evt = new int[10000]; + boolean fitsBuffers = rapidyaml.parseYs(srcbuf, arena, evt); + int reqsizeInts = rapidyaml.requiredSizeEvt(); + int reqsizeArena = rapidyaml.requiredSizeArena(); + if(!fitsBuffers) { + if(reqsizeInts > evt.length) evt = new int[reqsizeInts]; + if(reqsizeArena > arena.length) throw new RuntimeException("resize arena"); + System.arraycopy(src, 0, srcbuf, 0, src.length); + boolean fitsBuffers2 = rapidyaml.parseYs(srcbuf, arena, evt); + int reqsizeInts2 = rapidyaml.requiredSizeEvt(); + int reqsizeArena2 = rapidyaml.requiredSizeArena(); + if(fitsBuffers2 != fitsBuffers) throw new RuntimeException("fitsBuffers"); + if(reqsizeInts2 != reqsizeInts) throw new RuntimeException("reqsizeInts"); + if(reqsizeArena2 != reqsizeArena) throw new RuntimeException("reqsizeArena"); + } + int[] ret = new int[reqsizeInts]; + System.arraycopy(evt, 0, ret, 0, reqsizeInts); + return ret; + } + + static IntBuffer callParseDirect(byte[] src, ByteBuffer srcbuf, ByteBuffer arena) throws Exception + { + Rapidyaml rapidyaml = new Rapidyaml(); + arena.position(0); + srcbuf.position(0); + srcbuf.put(src); + IntBuffer evt = Rapidyaml.mkIntBuffer(10000); + boolean fitsBuffers = rapidyaml.parseYsDirect(srcbuf, arena, evt); + int reqsizeInts = rapidyaml.requiredSizeEvt(); + int reqsizeArena = rapidyaml.requiredSizeArena(); + if(!fitsBuffers) { + if(reqsizeInts > srcbuf.position()) evt = Rapidyaml.mkIntBuffer(reqsizeInts); + if(reqsizeArena > arena.capacity()) throw new RuntimeException("resize arena"); + arena.position(0); + srcbuf.position(0); + srcbuf.put(src); + boolean fitsBuffers2 = rapidyaml.parseYsDirect(srcbuf, arena, evt); + int reqsizeInts2 = rapidyaml.requiredSizeEvt(); + int reqsizeArena2 = rapidyaml.requiredSizeArena(); + if(fitsBuffers2 != fitsBuffers) throw new RuntimeException("fitsBuffers"); + if(reqsizeInts2 != reqsizeInts) throw new RuntimeException("reqsizeInts"); + if(reqsizeArena2 != reqsizeArena) throw new RuntimeException("reqsizeArena"); + } + evt.position(reqsizeInts); + return evt; + } + + ExpectedEvent mkev(int flags) + { + return new ExpectedEvent(flags); + } + + ExpectedEvent mkev(int flags, int offs, int len, String ref) + { + return new ExpectedEvent(flags, offs, len, ref); + } +} + +// the result is an array of integers, but we use this to simplify +// running the tests +class ExpectedEvent +{ + int flags; + int str_start; + int str_len; + String str; + ExpectedEvent(int flags) + { + this.flags = flags; + this.str_start = 0; + this.str_len = 0; + this.str = ""; + } + ExpectedEvent(int flags, int str_start, int str_len, String str) + { + this.flags = flags; + this.str_start = str_start; + this.str_len = str_len; + this.str = str; + } + int required_size() + { + return ((flags & Evt.WSTR) != 0) ? 3 : 1; + } + + public static int required_size_(ExpectedEvent[] evts) + { + int sz = 0; + for(int i = 0; i < evts.length; ++i) { + sz += evts[i].required_size(); + } + return sz; + } +}; diff --git a/ruby/ReadMe.md b/ruby/ReadMe.md index 512b7ed15..f25a1947b 100644 --- a/ruby/ReadMe.md +++ b/ruby/ReadMe.md @@ -126,7 +126,7 @@ $ ruby prog.rb You can install this module like any other Ruby module: ```bash -$ gem install yamlscript +gem install yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -134,7 +134,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/rust/ReadMe.md b/rust/ReadMe.md index 01ac18f8b..be61bd845 100644 --- a/rust/ReadMe.md +++ b/rust/ReadMe.md @@ -146,7 +146,7 @@ Object {"bar": Object {"oh": String("Hello")}, "baz": String("Hello, World!"), " You can install this module like any other Rust module: ```bash -$ cargo add yamlscript +cargo add yamlscript ``` but you will need to have a system install of `libyamlscript.so`. @@ -154,7 +154,7 @@ but you will need to have a system install of `libyamlscript.so`. One simple way to do that is with: ```bash -$ curl https://yamlscript.org/install | bash +curl https://yamlscript.org/install | bash ``` > Note: The above command will install the latest version of the YAMLScript diff --git a/util/RYS b/util/RYS new file mode 100755 index 000000000..1d53e01a0 --- /dev/null +++ b/util/RYS @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +( + root=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd -P) + root_ry=$root/rapidyaml + make --no-print-directory -C "$root_ry" build + export YS_PARSER_RAPIDYAML=1 + export LD_LIBRARY_PATH=$root_ry/native + bin=$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P) + exec "$bin/YS" "$@" +) diff --git a/util/YS b/util/YS index 9e7184ca3..53eff8a5f 100755 --- a/util/YS +++ b/util/YS @@ -10,6 +10,6 @@ set -euo pipefail export PATH=$JAVA_HOME/bin:$PATH root_ys=$root/ys jar=yamlscript.cli-$libyamlscript_version-SNAPSHOT-standalone.jar - make --no-print-directory -C "$root_ys" jar + make --quiet --no-print-directory -C "$root_ys" jar java -jar "$root_ys/target/uberjar/$jar" "$@" ) diff --git a/www/src/install b/www/src/install index 818f1a884..9a3ebf00b 100644 --- a/www/src/install +++ b/www/src/install @@ -9,8 +9,8 @@ set -e -u -o pipefail -LIBYAMLSCRIPT_VERSION=0.1.95 -VERSION=${VERSION:-$LIBYAMLSCRIPT_VERSION} +YAMLSCRIPT_VERSION=0.1.95 +VERSION=${VERSION:-$YAMLSCRIPT_VERSION} export VERSION main() ( diff --git a/ys/Makefile b/ys/Makefile index 8044f4cbf..a2e6537b6 100644 --- a/ys/Makefile +++ b/ys/Makefile @@ -1,77 +1,52 @@ include ../common/base.mk include $(COMMON)/java.mk include $(COMMON)/clojure.mk -include $(COMMON)/docker.mk include $(COMMON)/native.mk - -YAMLSCRIPT_CLI_BIN := bin/ys-$(YS_VERSION) -YAMLSCRIPT_CLI_SRC := \ - src/yamlscript/cli.clj \ - -YAMLSCRIPT_CLI_BIN_BASH := bin/ys-sh-$(YS_VERSION) -YAMLSCRIPT_CLI_BIN_BASH_SRC := share/ys-0.bash - -YAMLSCRIPT_CLI_JAR_PATH := \ - target/uberjar/yamlscript.cli-$(YS_VERSION)-SNAPSHOT-standalone.jar - -# Avoid rebuild in ephemerally created Docker container. -ifdef DOCKERENV -ifneq (,$(wildcard $(YAMLSCRIPT_CLI_BIN))) -YAMLSCRIPT_CLI_JAR_PATH := -endif -endif +include $(COMMON)/vars-cli.mk BPAN_LOCAL := .bpan BPAN_REPO_URL := https://github.com/bpan-org/bpan test := test/ -YS_BUILD_DEPS := \ - $(YAMLSCRIPT_CLI_BIN) \ - $(YAMLSCRIPT_CLI_BIN_BASH) \ - -ifeq (true,$(IS_LINUX)) -ifeq (true,$(IS_INTEL)) -YS_BUILD_DEPS := $(MUSL_GCC) $(YS_BUILD_DEPS) -NATIVE_OPTS += \ - -H:CCompilerOption=-Wl,-z,stack-size=2097152 \ - --static \ - --libc=musl -endif -endif - #------------------------------------------------------------------------------ -build:: $(YS_BUILD_DEPS) +build:: $(CLI_DEPS) -install: build +install: $(CLI_DEPS) mkdir -p $(PREFIX)/bin - install -m 755 $(YAMLSCRIPT_CLI_BIN) $(PREFIX)/bin/ - ln -fs $(notdir $(YAMLSCRIPT_CLI_BIN)) \ - $(PREFIX)/$(YAMLSCRIPT_CLI_BIN:%-$(YS_VERSION)=%-$(API_VERSION)) - ln -fs $(notdir $(YAMLSCRIPT_CLI_BIN)) \ - $(PREFIX)/$(YAMLSCRIPT_CLI_BIN:%-$(YS_VERSION)=%) - install -m 755 $(YAMLSCRIPT_CLI_BIN_BASH) \ + install -m 755 $(CLI_BIN) $(PREFIX)/bin/ + ln -fs $(notdir $(CLI_BIN)) \ + $(PREFIX)/$(CLI_BIN:%-$(YS_VERSION)=%-$(API_VERSION)) + ln -fs $(notdir $(CLI_BIN)) \ + $(PREFIX)/$(CLI_BIN:%-$(YS_VERSION)=%) + install -m 755 $(CLI_BIN_BASH) \ $(PREFIX)/bin/ -jar: $(YAMLSCRIPT_CLI_JAR_PATH) - @: +jar: $(CLI_JAR) -test: test-unit +test: test-run test-all: test-unit test-run -test-unit: $(LEIN) $(YAMLSCRIPT_CORE_INSTALLED) - $< test $(lein-test) +test-unit: $(CORE_INSTALLED) + $(LEIN) test $(lein-test) -test-run: $(BPAN_LOCAL) build +test-run: $(CLI_DEPS) $(BPAN_LOCAL) prove $${TEST_VERBOSE:+'-v'} $(test) +repl-deps:: $(CORE_INSTALLED) + +clean:: + $(RM) -r .cpcache/ + +realclean:: + $(RM) -r bin lib $(BPAN_LOCAL) + $(BPAN_LOCAL): git clone --depth=1 $(BPAN_REPO_URL) $@ -$(YAMLSCRIPT_CLI_BIN): $(YAMLSCRIPT_CLI_JAR_PATH) -ifndef YS_BIN_DIR +$(CLI_BIN): $(CLI_JAR) ifneq (true,$(LIBZ)) $(error *** \ The 'libz.$(SO)' library is required by native-image but not installed) @@ -86,34 +61,21 @@ endif -o $@ ln -fs $(notdir $@) $(@:%-$(YS_VERSION)=%-$(API_VERSION)) ln -fs $(notdir $@) $(@:%-$(YS_VERSION)=%) -endif + touch $@ -$(YAMLSCRIPT_CLI_BIN_BASH): $(YAMLSCRIPT_CLI_BIN_BASH_SRC) +$(CLI_BIN_BASH): $(CLI_BIN_BASH_SRC) mkdir -p $(dir $@) cp $< $@ chmod 755 $@ -clean:: - $(RM) -r .cpcache/ +$(CLI_JAR): $(CLI_JAR_DEPS) + $(LEIN) uberjar + jar -uf $@ \ + -C $(RAPIDYAML_DIR)/native/ librapidyaml.0.8.0.so + touch $@ -realclean:: - $(RM) -r bin lib $(BPAN_LOCAL) - -ifeq (true,$(IS_ROOT)) -$(YAMLSCRIPT_CLI_JAR_PATH): -else -$(YAMLSCRIPT_CLI_JAR_PATH): $(LEIN) $(YAMLSCRIPT_CORE_INSTALLED) $(YAMLSCRIPT_CLI_SRC) - $< uberjar -endif +$(CORE_INSTALLED): + $(MAKE) -C $(CORE_DIR) $@ -Dockerfile:: $(COMMON) Makefile - cat \ - $ $@ +$(CORE_JAR): + $(MAKE) -C $(CORE_DIR) $@ diff --git a/ys/config/reflect-config.json b/ys/config/reflect-config.json new file mode 120000 index 000000000..c0189b67e --- /dev/null +++ b/ys/config/reflect-config.json @@ -0,0 +1 @@ +../../common/reflection.json \ No newline at end of file diff --git a/ys/config/resource-config.json b/ys/config/resource-config.json new file mode 100644 index 000000000..0bc3aa5c7 --- /dev/null +++ b/ys/config/resource-config.json @@ -0,0 +1,6 @@ +{ + "resources": [ + {"pattern": "librapidyaml.so"}, + {"pattern": "librapibyaml.dylib"} + ] +}