diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2670d87959f..d851d7e9b7b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,8 +1,8 @@ ARG COUCHDB_IMAGE FROM ${COUCHDB_IMAGE} -# Install SpiderMonkey 78 and tell CouchDB to use it in configure -ENV SM_VSN=78 +# Install SpiderMonkey 128 and tell CouchDB to use it in configure +ENV SM_VSN=128 USER root diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index eb0831cf5b2..986959e5602 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,7 @@ // apache/couchdbci-debian:bullseye-erlang-25.3.2.7 // apache/couchdbci-debian:bookworm-erlang-24.3.4.14 // - "COUCHDB_IMAGE": "apache/couchdbci-debian:bullseye-erlang-24.3.4.14" + "COUCHDB_IMAGE": "apache/couchdbci-debian:trixie-erlang-26.2.5.16" } }, @@ -28,7 +28,7 @@ "customizations": { "vscode": { "extensions": [ - "erlang-ls.erlang-ls", + "erlang-language-platform.erlang-language-platform", "redhat.java" ] } diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 360d4fa6226..00000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: Bug report -about: Describe a scenario in which CouchDB behaves unexpectedly -title: '' -labels: bug, needs-triage -assignees: '' - ---- - -[NOTE]: # ( ^^ Provide a general summary of the issue in the title above. ^^ ) - -## Description - -[NOTE]: # ( Describe the problem you're encountering. ) -[TIP]: # ( Do NOT give us access or passwords to your actual CouchDB! ) - -## Steps to Reproduce - -[NOTE]: # ( Include commands to reproduce, if possible. curl is preferred. ) - -## Expected Behaviour - -[NOTE]: # ( Tell us what you expected to happen. ) - -## Your Environment - -[TIP]: # ( Include as many relevant details about your environment as possible. ) -[TIP]: # ( You can paste the output of curl http://YOUR-COUCHDB:5984/ here. ) - -* CouchDB version used: -* Browser name and version: -* Operating system and version: - -## Additional Context - -[TIP]: # ( Add any other context about the problem here. ) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000000..80f76f12198 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,60 @@ +name: Bug report +description: Describe a scenario in which CouchDB behaves unexpectedly +title: "[Bug]: " +labels: ["bug", "needs-triage"] +body: + - type: dropdown + id: version + attributes: + label: Version + description: What version of our software are you running? + options: + - unknown + - 3.0.0 + - 3.0.1 + - 3.1.0 + - 3.1.1 + - 3.1.2 + - 3.2.0 + - 3.2.1 + - 3.2.2 + - 3.2.3 + - 3.3.0 + - 3.3.1 + - 3.3.2 + - 3.3.3 + - 3.4.0 + - 3.4.1 + - 3.4.2 + - 3.4.3 + - 3.5.0 + - 3.5.1 + default: 0 + - type: textarea + id: summary + attributes: + label: Describe the problem you're encountering + validations: + required: true + - type: textarea + id: expected-behaviour + attributes: + label: Expected Behaviour + description: Tell us what you expected to happen + validations: + required: true + - type: textarea + id: repro-steps + attributes: + label: Steps to Reproduce + description: Include commands to reproduce, if possible. curl is preferred + - type: textarea + id: environment + attributes: + label: Your Environment + description: Include as many relevant details about your environment as possible + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context about the problem here diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..3ba13e0cec6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md deleted file mode 100644 index ca92725a6bd..00000000000 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Enhancement request -about: Suggest an idea for a future version of CouchDB -title: '' -labels: enhancement, needs-triage -assignees: '' - ---- - -[NOTE]: # ( ^^ Provide a general summary of the request in the title above. ^^ ) - -## Summary - -[NOTE]: # ( Provide a brief overview of what the new feature is all about. ) - -## Desired Behaviour - -[NOTE]: # ( Tell us how the new feature should work. Be specific. ) -[TIP]: # ( Do NOT give us access or passwords to your actual CouchDB! ) - -## Possible Solution - -[NOTE]: # ( Not required. Suggest how to implement the addition or change. ) - -## Additional context - -[TIP]: # ( Why does this feature matter to you? What unique circumstances do you have? ) diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml new file mode 100644 index 00000000000..7e24c425655 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.yml @@ -0,0 +1,22 @@ +name: Enhancement request +description: Suggest an idea for a future version of CouchDB +title: '[ENHANCEMENT]: ' +labels: ["enhancement", "needs-triage"] +body: + - type: textarea + id: summary + attributes: + label: Provide a brief overview of what the new feature is all about + - type: textarea + id: desired-behaviour + attributes: + label: Tell us how the new feature should work. Be specific + - type: textarea + id: possible-solution + attributes: + label: Not required. Suggest how to implement the addition or change + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Why does this feature matter to you? What unique circumstances do you have? diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md deleted file mode 100644 index a966bd9f80e..00000000000 --- a/.github/ISSUE_TEMPLATE/rfc.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: Formal RFC -about: Submit a formal Request For Comments for consideration by the team. -title: '' -labels: rfc, discussion -assignees: '' - ---- - -[NOTE]: # ( ^^ Provide a general summary of the RFC in the title above. ^^ ) - -# Introduction - -## Abstract - -[NOTE]: # ( Provide a 1-to-3 paragraph overview of the requested change. ) -[NOTE]: # ( Describe what problem you are solving, and the general approach. ) - -## Requirements Language - -[NOTE]: # ( Do not alter the section below. Follow its instructions. ) - -The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", -"SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this -document are to be interpreted as described in -[RFC 2119](https://www.rfc-editor.org/rfc/rfc2119.txt). - -## Terminology - -[TIP]: # ( Provide a list of any unique terms or acronyms, and their definitions here.) - ---- - -# Detailed Description - -[NOTE]: # ( Describe the solution being proposed in greater detail. ) -[NOTE]: # ( Assume your audience has knowledge of, but not necessarily familiarity ) -[NOTE]: # ( with, the CouchDB internals. Provide enough context so that the reader ) -[NOTE]: # ( can make an informed decision about the proposal. ) - -[TIP]: # ( Artwork may be attached to the submission and linked as necessary. ) -[TIP]: # ( ASCII artwork can also be included in code blocks, if desired. ) - -# Advantages and Disadvantages - -[NOTE]: # ( Briefly, list the benefits and drawbacks that would be realized should ) -[NOTE]: # ( the proposal be accepted for inclusion into Apache CouchDB. ) - -# Key Changes - -[TIP]: # ( If the changes will affect how a user interacts with CouchDB, explain. ) - -## Applications and Modules affected - -[NOTE]: # ( List the OTP applications or functional modules in CouchDB affected by the proposal. ) - -## HTTP API additions - -[NOTE]: # ( Provide *exact* detail on each new API endpoint, including: ) -[NOTE]: # ( HTTP methods [HEAD, GET, PUT, POST, DELETE, etc.] ) -[NOTE]: # ( Synopsis of functionality ) -[NOTE]: # ( Headers and parameters accepted ) -[NOTE]: # ( JSON in [if a PUT or POST type] ) -[NOTE]: # ( JSON out ) -[NOTE]: # ( Valid status codes and their definitions ) -[NOTE]: # ( A proposed Request and Response block ) - -## HTTP API deprecations - -[NOTE]: # ( Provide *exact* detail on the API endpoints to be deprecated. ) -[NOTE]: # ( If these endpoints are replaced by new endpoints, list those as well. ) -[NOTE]: # ( State the proposed version in which the deprecation and removal will occur. ) - -# Security Considerations - -[NOTE]: # ( Include any impact to the security of CouchDB here. ) - -# References - -[TIP]: # ( Include any references to CouchDB documentation, mailing list discussion, ) -[TIP]: # ( external standards or other links here. ) - -# Acknowledgements - -[TIP]: # ( Who helped you write this RFC? ) diff --git a/.github/ISSUE_TEMPLATE/rfc.yml b/.github/ISSUE_TEMPLATE/rfc.yml new file mode 100644 index 00000000000..64359e13614 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.yml @@ -0,0 +1,93 @@ +name: Formal RFC +description: Submit a formal Request For Comments for consideration by the team. +title: '[RFC]: ' +labels: ["rfc", "discussion"] +body: + - type: markdown + attributes: + value: | + ## Requirements Language + + The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + document are to be interpreted as described in + [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119.txt). + - type: textarea + id: abstract + attributes: + label: Provide a 1-to-3 paragraph overview of the requested change. + - type: textarea + id: terminology + attributes: + label: Terminology + - type: textarea + id: detailed-description + attributes: + label: Detailed Description + description: | + Describe the solution being proposed in greater detail. Assume your + audience has knowledge of, but not necessarily familiarity with, the + CouchDB internals. Provide enough context so that the reader can make an + informed decision about the proposal. + + Artwork may be attached to the submission and linked as necessary. + ASCII artwork can also be included in code blocks, if desired. + - type: textarea + id: advantages-and-disadvantages + attributes: + label: Advantages and Disadvantages + description: | + Briefly, list the benefits and drawbacks that would be realized should + the proposal be accepted for inclusion into Apache CouchDB. + - type: textarea + id: key-changes + attributes: + label: Key Changes + description: If the changes will affect how a user interacts with CouchDB, explain + - type: textarea + id: applications-and-modules-affected + attributes: + label: Applications and Modules affected + description: List the OTP applications or functional modules in CouchDB affected by the proposal + - type: textarea + id: http-api-additions + attributes: + label: HTTP API additions + description: | + Provide *exact* detail on each new API endpoint, including: + HTTP methods [HEAD, GET, PUT, POST, DELETE, etc.] + Synopsis of functionality + Headers and parameters accepted + JSON in [if a PUT or POST type] + JSON out + Valid status codes and their definitions + A proposed Request and Response block + placeholder: None + - type: textarea + id: http-api-deprecations + attributes: + label: HTTP API deprecations + description: | + Provide *exact* detail on the API endpoints to be deprecated. + If these endpoints are replaced by new endpoints, list those as well. + State the proposed version in which the deprecation and removal will occur. + placeholder: None + - type: textarea + id: security-considerations + attributes: + label: Security Considerations + description: Include any impact to the security of CouchDB here + placeholder: None + - type: textarea + id: references + attributes: + label: References + description: | + Include any references to CouchDB documentation, mailing list discussion, + external standards or other links here. + placeholder: None + - type: textarea + id: acknowledgements + attributes: + label: Acknowledgements + placeholder: None diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 68fa0c5f2ee..de2462ce8f5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,7 +9,16 @@ to proceed. Once there are no objections, the PR can be merged by a CouchDB committer. - See: http://couchdb.apache.org/bylaws.html#decisions for more info. --> + See: http://couchdb.apache.org/bylaws.html#decisions for more info. + + Artificial Intelligence and Large Language Models Contributions Policy + + It is expressly forbidden to contribute material generated by + AI, LLMs, and similar technologies, to the CouchDB project. + This includes, but is not limited to, source code, documentation, + commit messages, or any other areas of the project. + +--> ## Overview @@ -29,6 +38,7 @@ ## Checklist +- [ ] This is my own work, I did not use AI, LLM's or similar technology - [ ] Code is written and works correctly - [ ] Changes are covered by tests - [ ] Any new configurable parameters are documented in `rel/overlay/etc/default.ini` diff --git a/.gitignore b/.gitignore index d46777512b7..28dcadf05f7 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,4 @@ test/javascript/junit.xml *.lock .tool-versions +mise.local.toml diff --git a/Makefile b/Makefile index 6847037df77..e046a099609 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ include version.mk REBAR?=$(CURDIR)/bin/rebar REBAR3?=$(CURDIR)/bin/rebar3 ERLFMT?=$(CURDIR)/bin/erlfmt -GRADLE?=$(CURDIR)/nouveau/gradlew +GRADLE?=$(CURDIR)/extra/nouveau/gradlew # Handle the following scenarios: # 1. When building from a tarball, use version.mk. @@ -75,7 +75,7 @@ DESTDIR= # Rebar options apps= -skip_deps=meck,mochiweb,triq,proper,snappy,ibrowse +skip_deps=meck,mochiweb,triq,proper,snappy,ibrowse,gun,recon suites= tests= @@ -106,7 +106,7 @@ endif .PHONY: all # target: all - Build everything -all: couch fauxton docs escriptize nouveau +all: couch-core fauxton docs escriptize extra/nouveau .PHONY: help @@ -123,9 +123,9 @@ help: ################################################################################ -.PHONY: couch -# target: couch - Build CouchDB core, use ERL_COMPILER_OPTIONS to provide custom compiler's options -couch: config.erl +.PHONY: couch-core +# target: couch-core - Build CouchDB core, use ERL_COMPILER_OPTIONS to provide custom compiler's options +couch-core: config.erl @COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) compile $(COMPILE_OPTS) ifeq ($(with_spidermonkey), true) @cp src/couch/priv/couchjs bin/ @@ -146,7 +146,7 @@ fauxton: share/www .PHONY: escriptize # target: escriptize - Build CLI tools -escriptize: couch +escriptize: couch-core @$(REBAR) -r escriptize apps=weatherreport @cp src/weatherreport/weatherreport bin/weatherreport @@ -168,40 +168,27 @@ check: all @$(MAKE) nouveau-test ifdef apps -subdirs = $(apps) +EUNIT_SUBDIRS = $(strip $(subst $(comma),$(space),$(apps))) else -subdirs=$(shell ls src) +EUNIT_SUBDIRS = $(filter-out fauxton docs, $(shell ls src)) endif -.PHONY: eunit # target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options -eunit: export BUILDDIR = $(CURDIR) +.PHONY: eunit $(EUNIT_SUBDIRS) eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -eunit: couch - @COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) setup_eunit 2> /dev/null - @for dir in $(subdirs); do \ - COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$$dir || exit 1; \ - done - - -setup-eunit: export BUILDDIR = $(CURDIR) -setup-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -setup-eunit: - @$(REBAR) setup_eunit 2> /dev/null - -just-eunit: export BUILDDIR = $(CURDIR) -just-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -just-eunit: - @$(REBAR) -r eunit $(EUNIT_OPTS) - -.PHONY: soak-eunit -soak-eunit: export BUILDDIR = $(CURDIR) -soak-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -soak-eunit: couch - @$(REBAR) setup_eunit 2> /dev/null - while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done +eunit: ${EUNIT_SUBDIRS} + +$(EUNIT_SUBDIRS): + @rm -rf tmp/$@ + @$(REBAR) setup_eunit app=$@ >/dev/null + @BUILDDIR=$(CURDIR)/tmp/$@ COUCHDB_VERSION=$(COUCHDB_VERSION) COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ && rm -rf tmp/$@ + +# cat together couch_log files after running eunit test +.PHONY: catlogs +catlogs: + @ls tmp/*/couch.log 2>/dev/null | xargs cat > tmp/couch.log || true # target: erlfmt-check - Check Erlang source code formatting erlfmt-check: @@ -284,7 +271,7 @@ ifneq ($(_WITH_CLOUSEAU), ) "$(_WITH_CLOUSEAU)" \ "$(TEST_OPTS)" \ --locald-config test/config/test-config.ini \ - --no-eval 'mix test --trace --include test/elixir/test/config/search.elixir' + --no-eval 'mix test --trace --include test/elixir/test/config/search.elixir $(EXUNIT_OPTS)' else @echo "Warning: Clouseau is not enabled, \`elixir-search\` cannot be run." endif @@ -303,6 +290,7 @@ build-report: build-aux/show-test-results.py --suites=10 --tests=10 > test-results.log || true cat ./dev/logs/node1.log || true cat ./dev/logs/nouveau.log || true + $(MAKE) catlogs || true cat ./tmp/couch.log || true cat test-results.log || true @@ -356,7 +344,7 @@ weatherreport-test: devclean escriptize .PHONY: quickjs-test262 # target: quickjs-javascript-tests - Run QuickJS JS conformance tests -quickjs-test262: couch +quickjs-test262: couch-core make -C src/couch_quickjs/quickjs test2-bootstrap make -C src/couch_quickjs/quickjs test2 @@ -450,8 +438,8 @@ endif ifeq ($(with_nouveau), true) @mkdir rel/couchdb/nouveau - @cd nouveau && $(GRADLE) installDist - @cp -R nouveau/build/install/nouveau rel/couchdb + @cd extra/nouveau && $(GRADLE) installDist + @cp -R extra/nouveau/build/install/nouveau rel/couchdb endif @echo "... done" @@ -500,7 +488,7 @@ clean: @rm -rf src/couch_dist/certs/out @rm -rf src/docs/build src/docs/.venv ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) clean + @cd extra/nouveau && $(GRADLE) clean endif @@ -565,12 +553,12 @@ derived: # Nouveau ################################################################################ -.PHONY: nouveau +.PHONY: extra/nouveau # target: nouveau - Build nouveau -nouveau: +extra/nouveau: ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) spotlessApply - @cd nouveau && $(GRADLE) build -x test + @cd extra/nouveau && $(GRADLE) spotlessApply + @cd extra/nouveau && $(GRADLE) build -x test endif .PHONY: nouveau-test @@ -578,15 +566,15 @@ endif nouveau-test: nouveau-test-gradle nouveau-test-elixir .PHONY: nouveau-test-gradle -nouveau-test-gradle: couch nouveau +nouveau-test-gradle: couch-core extra/nouveau ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) test --info --rerun + @cd extra/nouveau && $(GRADLE) test --info --rerun endif .PHONY: nouveau-test-elixir nouveau-test-elixir: export MIX_ENV=integration nouveau-test-elixir: elixir-init devclean -nouveau-test-elixir: couch nouveau +nouveau-test-elixir: couch-core extra/nouveau ifeq ($(with_nouveau), true) @dev/run "$(TEST_OPTS)" -n 1 -q -a adm:pass --with-nouveau \ --locald-config test/config/test-config.ini \ diff --git a/Makefile.win b/Makefile.win index d510ee51df4..1482472fdb0 100644 --- a/Makefile.win +++ b/Makefile.win @@ -22,7 +22,7 @@ REBAR?=$(CURDIR)/bin/rebar.cmd PYTHON=python.exe ERLFMT?=$(CURDIR)/bin/erlfmt.cmd MAKE=make -f Makefile.win -GRADLE?=$(CURDIR)/nouveau/gradlew.bat +GRADLE?=$(CURDIR)/extra/nouveau/gradlew.bat # REBAR?=$(shell where rebar.cmd) # Handle the following scenarios: @@ -80,7 +80,7 @@ DESTDIR= # Rebar options apps= -skip_deps=meck,mochiweb,triq,proper,snappy,ibrowse,local +skip_deps=meck,mochiweb,triq,proper,snappy,ibrowse,gun,recon suites= tests= @@ -104,7 +104,7 @@ endif .PHONY: all # target: all - Build everything -all: couch fauxton docs nouveau +all: couch-core fauxton docs extra/nouveau .PHONY: help @@ -118,9 +118,9 @@ help: ################################################################################ -.PHONY: couch -# target: couch - Build CouchDB core, use ERL_COMPILER_OPTIONS to provide custom compiler's options -couch: config.erl +.PHONY: couch-core +# target: couch-core - Build CouchDB core, use ERL_COMPILER_OPTIONS to provide custom compiler's options +couch-core: config.erl @set COUCHDB_VERSION=$(COUCHDB_VERSION) && set COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) && $(REBAR) compile $(COMPILE_OPTS) ifeq ($(with_spidermonkey), true) @copy src\couch\priv\couchjs.exe bin @@ -155,30 +155,27 @@ check: all @$(MAKE) nouveau-test ifdef apps -subdirs=$(apps) +EUNIT_SUBDIRS = $(strip $(subst $(comma),$(space),$(apps))) else -subdirs=$(shell dir /b src) +EUNIT_SUBDIRS = $(filter-out fauxton docs, $(shell dir /b src)) endif -.PHONY: eunit # target: eunit - Run EUnit tests, use EUNIT_OPTS to provide custom options -eunit: export BUILDDIR = $(CURDIR) +.PHONY: eunit $(EUNIT_SUBDIRS) eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config eunit: export COUCHDB_QUERY_SERVER_JAVASCRIPT = $(CURDIR)/bin/couchjs $(CURDIR)/share/server/main.js eunit: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 -eunit: couch - @set COUCHDB_VERSION=$(COUCHDB_VERSION) && set COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) && $(REBAR) setup_eunit 2> nul - @cmd /c "FOR %d IN ($(subdirs)) DO set COUCHDB_VERSION=$(COUCHDB_VERSION) & set COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) & $(REBAR) -r eunit $(EUNIT_OPTS) apps=%d || exit /b 1" +eunit: $(EUNIT_SUBDIRS) -setup-eunit: export BUILDDIR = $(CURDIR) -setup-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -setup-eunit: - @$(REBAR) setup_eunit 2> nul +$(EUNIT_SUBDIRS): + -@rmdir /s /q tmp\$@ + @$(REBAR) setup_eunit app=$@ + @set BUILDDIR=$(CURDIR)/tmp/$@ && set COUCHDB_VERSION=$(COUCHDB_VERSION) && set COUCHDB_GIT_SHA=$(COUCHDB_GIT_SHA) && $(REBAR) -r eunit $(EUNIT_OPTS) apps=$@ && @rmdir /s/q tmp\$@ -just-eunit: export BUILDDIR = $(CURDIR) -just-eunit: export ERL_AFLAGS = -config $(CURDIR)/rel/files/eunit.config -just-eunit: - @$(REBAR) -r eunit $(EUNIT_OPTS) +# cat together couch_log files after running eunit test +.PHONY: catlogs +catlogs: + @type tmp/*/couch.log > tmp/couch.log || true # target: erlfmt-check - Check Erlang source code formatting erlfmt-check: export ERLFMT_PATH = $(ERLFMT) @@ -241,7 +238,6 @@ elixir-cluster-with-quorum: elixir-init devclean .PHONY: elixir # target: elixir - Run Elixir-based integration tests elixir: export MIX_ENV=integration -elixir: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 elixir: elixir-init devclean @dev\run "$(TEST_OPTS)" -n 1 -q -a adm:pass \ --enable-erlang-views \ @@ -284,6 +280,7 @@ build-report: @$(PYTHON) build-aux/show-test-results.py --suites=10 --tests=10 > test-results.log || true cat .\dev\logs\node1.log || true cat .\dev\logs\nouveau.log || true + $(MAKE) catlogs || true cat .\tmp\couch.log || true cat test-results.log || true @@ -305,7 +302,6 @@ list-eunit-suites: .PHONY: mango-test # target: mango-test - Run Mango tests -mango-test: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1 mango-test: devclean all @$(PYTHON) -m venv src\mango\.venv && \ src\mango\.venv\Scripts\pip.exe install -r src\mango\requirements.txt @@ -401,8 +397,8 @@ endif ifeq ($(with_nouveau), true) -@mkdir rel\couchdb\nouveau - @cd nouveau && $(GRADLE) installDist - @xcopy nouveau\build\install\nouveau rel\couchdb\nouveau /E /I + @cd extra\nouveau && $(GRADLE) installDist + @xcopy extra\nouveau\build\install\nouveau rel\couchdb\nouveau /E /I endif @echo ... done @@ -448,7 +444,7 @@ clean: -@del /f/q dev\boot_node.beam dev\pbkdf2.pyc log\crash.log >NUL 2>&1 || true -@rmdir /s/q src\docs\build src\docs\.venv >NUL 2>&1 || true ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) clean + @cd extra\nouveau && $(GRADLE) clean endif .PHONY: distclean @@ -491,8 +487,8 @@ config.erl: src\docs\build: - @echo 'Building docs...' ifeq ($(with_docs), true) + @echo 'Building docs...' @cd src\docs && setup.bat && make.bat html && make.bat man endif @@ -519,12 +515,12 @@ derived: # Nouveau ################################################################################ -.PHONY: nouveau +.PHONY: extra/nouveau # target: nouveau - Build nouveau -nouveau: +extra/nouveau: ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) spotlessApply - @cd nouveau && $(GRADLE) build -x test + @cd extra/nouveau && $(GRADLE) spotlessApply + @cd extra/nouveau && $(GRADLE) build -x test endif .PHONY: nouveau-test @@ -532,16 +528,15 @@ endif nouveau-test: nouveau-test-gradle nouveau-test-elixir .PHONY: nouveau-test-gradle -nouveau-test-gradle: couch nouveau +nouveau-test-gradle: couch-core extra/nouveau ifeq ($(with_nouveau), true) - @cd nouveau && $(GRADLE) test --info --rerun - + @cd extra/nouveau && $(GRADLE) test --info --rerun endif .PHONY: nouveau-test-elixir nouveau-test-elixir: export MIX_ENV=integration nouveau-test-elixir: elixir-init devclean -nouveau-test-elixir: couch nouveau +nouveau-test-elixir: couch-core extra/nouveau ifeq ($(with_nouveau), true) @dev\run "$(TEST_OPTS)" -n 1 -q -a adm:pass --with-nouveau \ --locald-config test/elixir/test/config/test-config.ini \ diff --git a/README-DEV.rst b/README-DEV.rst index a2eec4b44f3..6726db0055a 100644 --- a/README-DEV.rst +++ b/README-DEV.rst @@ -223,6 +223,22 @@ to specify exact ``rebar eunit`` options:: make eunit EUNIT_OPTS="apps=couch,chttpd" +Running Erlang tests in Parallel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have GNU make (`gmake`) installed, you can supply a `-jN` parameter to +run as many jobs in parallel. To avoid clobbering output, use the +`--output-sync` option. + + gmake eunit -j2 --output-sync=target + +This runs two test modules in parallel at a time. On a machine with two or more +CPUs, this should reduce the total wall clock time of testing to up to one half. + +Higher `N` might also work, but the test suite is still undergoing hardening to +support full parallelism, so spurious errors that do not occur with `-j1` might +show up. + Elixir Integration Tests ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build-aux/Jenkinsfile b/build-aux/Jenkinsfile index 9bea62863bc..b1f0b8d6f60 100644 --- a/build-aux/Jenkinsfile +++ b/build-aux/Jenkinsfile @@ -30,7 +30,8 @@ MAXIMUM_ERLANG_VERSION = '28.1.1' // Use these to detect if just documents changed docs_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^src/docs/'" -other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v '^src/docs/'" +github_changed = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q '^.github'" +other_changes = "git diff --name-only origin/${env.CHANGE_TARGET} | grep -q -v -e '^src/docs/' -e '^.github'" // We create parallel build / test / package stages for each OS using the metadata // in this map. Adding a new OS should ideally only involve adding a new entry here. @@ -39,7 +40,7 @@ meta = [ name: 'CentOS 8', spidermonkey_vsn: '60', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/usr', quickjs_test262: true, image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" @@ -49,7 +50,7 @@ meta = [ name: 'CentOS 9', spidermonkey_vsn: '78', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/usr', quickjs_test262: true, image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" @@ -59,7 +60,7 @@ meta = [ name: 'Ubuntu 22.04', spidermonkey_vsn: '91', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" @@ -69,7 +70,7 @@ meta = [ name: 'Ubuntu 24.04', spidermonkey_vsn: '115', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" @@ -79,7 +80,7 @@ meta = [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" @@ -122,7 +123,7 @@ meta = [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', // Test this in in the bookworm-quickjs variant quickjs_test262: false, @@ -133,7 +134,7 @@ meta = [ name: 'Debian x86_64', spidermonkey_vsn: '78', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: false, image: "${DOCKER_IMAGE_BASE}-${MAXIMUM_ERLANG_VERSION}" @@ -143,7 +144,7 @@ meta = [ name: 'Debian 12 with QuickJS', disable_spidermonkey: true, with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" @@ -161,7 +162,7 @@ meta = [ name: 'Debian ARM64', spidermonkey_vsn: '78', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', // Test this in in the bookworm-quickjs variant quickjs_test262: false, @@ -173,7 +174,7 @@ meta = [ name: 'Debian x86_64', spidermonkey_vsn: '128', with_nouveau: true, - with_clouseau: true, + with_clouseau: false, clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-debian:trixie-erlang-${ERLANG_VERSION}" @@ -190,35 +191,34 @@ meta = [ // Spidermonkey 91 has issues on ARM64 FreeBSD // use QuickJS for now - 'freebsd-arm64': [ - name: 'FreeBSD ARM64 QuickJS', + // Temporarily disabled because on too slow a host + // 'freebsd-arm64': [ + // name: 'FreeBSD ARM64 QuickJS', + // disable_spidermonkey: true, + // with_clouseau: true, + // clouseau_java_home: '/usr/local/openjdk21', + // quickjs_test262: false, + // gnu_make: 'gmake' + // ], + + 'macos': [ + name: 'macOS', disable_spidermonkey: true, + with_nouveau: false, + with_clouseau: true, + clouseau_java_home: '/opt/homebrew/opt/openjdk@21', + gnu_make: 'gmake', + gnu_make_eunit_opts: '-j4 --output-sync=target' + ], + + 'win2022': [ + name: 'Windows 2022', + spidermonkey_vsn: '128', with_clouseau: true, - clouseau_java_home: '/usr/local/openjdk21', + clouseau_java_home: /C:\tools\zulu21.46.19-ca-jdk21.0.9-win_x64/, quickjs_test262: false, - gnu_make: 'gmake' - ], - - // Disable temporarily. Forks / shell execs seem to fail there currently - // - // - // 'macos': [ - // name: 'macOS', - // spidermonkey_vsn: '128', - // with_nouveau: false, - // with_clouseau: true, - // clouseau_java_home: '/opt/java/openjdk8/zulu-8.jre/Contents/Home', - // gnu_make: 'make' - // ], - - 'win2022': [ - name: 'Windows 2022', - spidermonkey_vsn: '128', - with_clouseau: true, - clouseau_java_home: /C:\tools\zulu21.46.19-ca-jdk21.0.9-win_x64/, - quickjs_test262: false, - node_label: 'win' - ] + node_label: 'win' + ] ] def String configure(config) { @@ -273,7 +273,11 @@ def generateNativeStage(platform) { dir( "${platform}/build" ) { sh "${configure(meta[platform])}" sh '$MAKE' - retry (3) {sh '$MAKE eunit'} + eunit_opts = '' + if ( meta[platform].gnu_make_eunit_opts ) { + eunit_opts = meta[platform].gnu_make_eunit_opts + } + retry (3) {sh '$MAKE eunit ' + eunit_opts} if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry (3) {sh '$MAKE elixir'} retry (3) {sh '$MAKE elixir-search ERLANG_COOKIE=crumbles'} @@ -321,58 +325,89 @@ def generateNativeStage(platform) { } powershell( script: """ - .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 + ..\\..\\couchdb-glazier\\bin\\shell.ps1 .\\configure.ps1 -SkipDeps -WithNouveau ${withClouseau} -SpiderMonkeyVersion ${meta[platform].spidermonkey_vsn} Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' make -f Makefile.win release """, label: 'Configure and Build') - //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win eunit", label: 'EUnit tests') - //powershell( script: ".\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win elixir", label: 'Elixir tests') + retry (3) { + powershell( script: """ + ..\\..\\couchdb-glazier\\bin\\shell.ps1 + Write-Output 'The following tests are skipped:' + ..\\..\\couchdb-glazier\\bin\\exclude_tests_win.ps1 -Path . | Out-Host + make -j 4 -f Makefile.win eunit + """, label: 'EUnit tests') + } - powershell( script: """ - .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 - Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' - ${setClouseauJavaHome} - make -f Makefile.win elixir-search ERLANG_COOKIE=crumbles - """, label: 'Clouseau tests') + retry (3) { + powershell( script: """ + ..\\..\\couchdb-glazier\\bin\\shell.ps1 + make -f Makefile.win elixir + """, label: 'Elixir tests') + } - powershell( script: """ - .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 - Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' - ${setClouseauJavaHome} - make -f Makefile.win mango-test ERLANG_COOKIE=crumbles - """, label: 'Mango tests') - - powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win weatherreport-test"', label: 'N/A Weatherreport tests') - - // temporary exclude - random flaky tests on Windows - //powershell( script: """ - // .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 - // Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' - // make -f Makefile.win nouveau-test - //""", label: 'Nouveau tests') - } + retry (3) { + timeout(time: 5, unit: "MINUTES") { + powershell( script: """ + ..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + ${setClouseauJavaHome} + make -f Makefile.win elixir-search ERLANG_COOKIE=crumbles + """, label: 'Clouseau tests') + } + } - powershell( script: """ - .\\couchdb-glazier\\bin\\shell.ps1 - .\\couchdb-glazier\\bin\\build_installer.ps1 -Path '${platform}/build' -IncludeGitSha -DisableICEChecks - """, label: 'Build Windows Installer file') + retry (3) { + timeout(time: 5, unit: "MINUTES") { + powershell( script: """ + ..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + ${setClouseauJavaHome} + make -f Makefile.win mango-test ERLANG_COOKIE=crumbles + """, label: 'Mango tests') + } + } - archiveArtifacts artifacts: '*.msi', fingerprint: true, onlyIfSuccessful: true + powershell( script: '..\\..\\couchdb-glazier\\bin\\shell.ps1; Write-Host "NOT AVAILABLE: make -f Makefile.win weatherreport-test"', label: 'N/A Weatherreport tests') + + retry (3) { + powershell( script: """ + ..\\..\\couchdb-glazier\\bin\\shell.ps1 + Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + make -f Makefile.win nouveau-test + """, label: 'Nouveau tests') + } + } } catch (err) { powershell( script: "Get-ChildItem ${WORKSPACE}") dir( "${platform}/build" ) { - powershell( script: '.\\..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win build-report') + powershell( script: '..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win build-report') powershell( script: 'Get-Content test-results.log') } error("Build step failed with error: ${err.getMessage()}") } finally { powershell( script: 'Get-ChildItem') + powershell( script: 'Get-Process erl -ErrorAction SilentlyContinue | Stop-Process -PassThru') + } + } + + stage("${meta[platform].name} - package") { + try { + powershell( script: """ + .\\couchdb-glazier\\bin\\shell.ps1 + .\\couchdb-glazier\\bin\\build_installer.ps1 -Path '${platform}/build' -IncludeGitSha -DisableICEChecks + """, label: 'Build Windows Installer file') + archiveArtifacts artifacts: '*.msi', fingerprint: true, onlyIfSuccessful: true + } + catch (err) { + powershell( script: "Get-ChildItem ${WORKSPACE}") + error("Build step failed with error: ${err.getMessage()}") + } + finally { powershell( script: "Remove-Item -Path '${WORKSPACE}\\*' -Force -Recurse -ErrorAction SilentlyContinue") - powershell( script: 'Get-ChildItem') } } } @@ -508,12 +543,20 @@ pipeline { script { env.DOCS_CHANGED = '0' env.ONLY_DOCS_CHANGED = '0' + env.GITHUB_CHANGED = '0' + env.ONLY_GITHUB_CHANGED = '0' if ( sh(returnStatus: true, script: docs_changed) == 0 ) { env.DOCS_CHANGED = '1' if (sh(returnStatus: true, script: other_changes) == 1) { env.ONLY_DOCS_CHANGED = '1' } } + if ( sh(returnStatus: true, script: github_changed) == 0 ) { + env.GITHUB_CHANGED = '1' + if (sh(returnStatus: true, script: other_changes) == 1) { + env.ONLY_GITHUB_CHANGED = '1' + } + } } } post { @@ -587,7 +630,7 @@ pipeline { stage('Source Format Checks') { when { beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } + expression { ONLY_DOCS_CHANGED == '0' && ONLY_GITHUB_CHANGED == '0' } } agent { docker { @@ -618,7 +661,7 @@ pipeline { stage('Build Release Tarball') { when { beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } + expression { ONLY_DOCS_CHANGED == '0' && ONLY_GITHUB_CHANGED == '0' } } agent { docker { @@ -632,7 +675,7 @@ pipeline { steps { sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' ) sh "./configure --spidermonkey-version 78 --with-nouveau" - sh 'make dist' + sh 'make -j4 dist' } post { success { @@ -652,12 +695,12 @@ pipeline { stage('Test and Package') { when { beforeOptions true - expression { ONLY_DOCS_CHANGED == '0' } + expression { ONLY_DOCS_CHANGED == '0' && ONLY_GITHUB_CHANGED == '0' } } steps { script { // Including failFast: true in map fails the build immediately if any parallel step fails - parallelStagesMap = meta.collectEntries( [failFast: true] ) { key, values -> + parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values -> if (values.image) { ["${key}": generateContainerStage(key)] } diff --git a/configure b/configure index f0433f5153b..819953da476 100755 --- a/configure +++ b/configure @@ -41,7 +41,7 @@ run_erlang() { COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)" JS_ENGINE=${JS_ENGINE:-"spidermonkey"} -SM_VSN=${SM_VSN:-"91"} +SM_VSN=${SM_VSN:-"128"} CLOUSEAU_MTH=${CLOUSEAU_MTH:-"dist"} CLOUSEAU_URI=${CLOUSEAU_URI:-"https://github.com/cloudant-labs/clouseau/releases/download/%s/clouseau-%s-dist.zip"} CLOUSEAU_VSN=${CLOUSEAU_VSN:-"3.0.0"} @@ -62,13 +62,15 @@ Options: -h | --help display a short help message and exit -u | --user USER set the username to run as (defaults to $COUCHDB_USER) + --dev alias for --disable-docs --disable-fauxton --disable-spidermonkey --disable-fauxton do not build Fauxton --disable-docs do not build any documentation or manpages + --disable-spidermonkey disable spidermonkey, don't try to build it + --spidermonkey-version VSN specify the version of SpiderMonkey to use (defaults to $SM_VSN) + --js-engine=ENGINE use js engine: spidermonkey or quickjs, defaults to spidermonkey --with-nouveau build the new experimental search module --with-clouseau build the Clouseau search module --erlang-md5 use erlang for md5 hash operations - --dev alias for --disable-docs --disable-fauxton - --spidermonkey-version VSN specify the version of SpiderMonkey to use (defaults to $SM_VSN) --clouseau-method MTH specify the method for Clouseau to deploy: git or dist (defaults to $CLOUSEAU_MTH) --clouseau-uri URI specify the location for retrieving Clouseau (defaults $(printf "$CLOUSEAU_URI" "$CLOUSEAU_VSN" "$CLOUSEAU_VSN")) --clouseau-version VSN specify the version (tag/branch for git) of Clouseau to use (defaults to $CLOUSEAU_VSN) @@ -76,8 +78,6 @@ Options: --rebar=PATH use rebar by specified path (version >=2.6.0 && <3.0 required) --rebar3=PATH use rebar3 by specified path --erlfmt=PATH use erlfmt by specified path - --js-engine=ENGINE use js engine: spidermonkey or quickjs, defaults to spidermonkey - --disable-spidermonkey disable spidermonkey, don't try to build it EOF } diff --git a/configure.ps1 b/configure.ps1 index 886136071f6..948a2414f73 100644 --- a/configure.ps1 +++ b/configure.ps1 @@ -12,7 +12,7 @@ -WithClouseau build the Clouseau search module (default false) -SkipDeps do not update Erlang dependencies (default false) -CouchDBUser USER set the username to run as (defaults to current user) - -SpiderMonkeyVersion VSN select the version of SpiderMonkey to use (default 91) + -SpiderMonkeyVersion VSN select the version of SpiderMonkey to use (default 128) -JSEngine ENGINE select JS engine to use (spidermonkey or quickjs) (default spidermonkey) -ClouseauVersion VSN select the version of Clouseau to use (default 2.25.0) -ClouseauMethod MTH method for Clouseau to deploy: git or dist (default dist) @@ -60,7 +60,7 @@ Param( [ValidateNotNullOrEmpty()] [string]$CouchDBUser = [Environment]::UserName, # set the username to run as (defaults to current user) [ValidateNotNullOrEmpty()] - [string]$SpiderMonkeyVersion = "91", # select the version of SpiderMonkey to use (default 91) + [string]$SpiderMonkeyVersion = "128", # select the version of SpiderMonkey to use (default 91) [ValidateNotNullOrEmpty()] [string]$JSEngine = "spidermonkey", # select the JS engine (spidermonkey | quickjs) to use (default spidermonkey) [ValidateNotNullOrEmpty()] diff --git a/dev/run b/dev/run index 640d15d22ff..9df9fd37686 100755 --- a/dev/run +++ b/dev/run @@ -546,7 +546,7 @@ def boot_nouveau(ctx): config = os.path.join(ctx["devdir"], "lib", "nouveau.yaml") gradle = "gradlew.bat" if os.name == "nt" else "gradlew" cmd = [ - os.path.join(ctx["rootdir"], "nouveau", gradle), + os.path.join(ctx["rootdir"], "extra", "nouveau", gradle), "run", "--args", f"server '{config}'", @@ -555,7 +555,7 @@ def boot_nouveau(ctx): log = open(logfname, "w") return sp.Popen( cmd, - cwd=os.path.join(ctx["rootdir"], "nouveau"), + cwd=os.path.join(ctx["rootdir"], "extra", "nouveau"), stdin=sp.PIPE, stdout=log, stderr=sp.STDOUT, @@ -909,6 +909,13 @@ def hack_default_ini(ctx, node, contents): flags=re.MULTILINE, ) + contents = re.sub( + r"^;uri_file =$", + "uri_file = ./dev/lib/{}/couch.uri".format(node), + contents, + flags=re.MULTILINE, + ) + if ctx["enable_erlang_views"]: contents = re.sub( r"^\[native_query_servers\]$", diff --git a/nouveau/.gitignore b/extra/nouveau/.gitignore similarity index 100% rename from nouveau/.gitignore rename to extra/nouveau/.gitignore diff --git a/nouveau/LICENSE b/extra/nouveau/LICENSE similarity index 100% rename from nouveau/LICENSE rename to extra/nouveau/LICENSE diff --git a/nouveau/README.md b/extra/nouveau/README.md similarity index 98% rename from nouveau/README.md rename to extra/nouveau/README.md index 2ebacd9612f..97825c82035 100644 --- a/nouveau/README.md +++ b/extra/nouveau/README.md @@ -3,8 +3,8 @@ Nouveau is a modern replacement for dreyfus/clouseau and is built on; 1) the Dropwizard framework (https://dropwizard.io) -2) Java 11+ -3) Lucene 9 +2) Java 21+ +3) Lucene 10 Nouveau transforms Apache CouchDB databases into Apache Lucene indexes at the shard level and then merges the results together. @@ -23,7 +23,7 @@ This work is currently EXPERIMENTAL and may change in ways that invalidate any e * integration with resharding * update=false * `_nouveau_info` -* `_search_cleanup` +* `_nouveau_cleanup` * /openapi.{json.yaml} ## What doesn't work yet? diff --git a/nouveau/TODO b/extra/nouveau/TODO similarity index 100% rename from nouveau/TODO rename to extra/nouveau/TODO diff --git a/nouveau/build.gradle b/extra/nouveau/build.gradle similarity index 97% rename from nouveau/build.gradle rename to extra/nouveau/build.gradle index a4de6ed7a8a..a91555971d4 100644 --- a/nouveau/build.gradle +++ b/extra/nouveau/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation 'io.dropwizard.metrics:metrics-jersey2' testImplementation 'io.dropwizard:dropwizard-testing' - def luceneVersion = '9.12.1' + def luceneVersion = '10.3.2' implementation 'org.apache.lucene:lucene-core:' + luceneVersion implementation 'org.apache.lucene:lucene-queryparser:' + luceneVersion implementation 'org.apache.lucene:lucene-analysis-common:' + luceneVersion @@ -46,7 +46,7 @@ group = 'org.apache.couchdb' version = '1.0-SNAPSHOT' java { - sourceCompatibility = "11" + sourceCompatibility = "21" } jar { diff --git a/nouveau/gradle/wrapper/gradle-wrapper.jar b/extra/nouveau/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from nouveau/gradle/wrapper/gradle-wrapper.jar rename to extra/nouveau/gradle/wrapper/gradle-wrapper.jar diff --git a/nouveau/gradle/wrapper/gradle-wrapper.properties b/extra/nouveau/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from nouveau/gradle/wrapper/gradle-wrapper.properties rename to extra/nouveau/gradle/wrapper/gradle-wrapper.properties diff --git a/nouveau/gradlew b/extra/nouveau/gradlew similarity index 100% rename from nouveau/gradlew rename to extra/nouveau/gradlew diff --git a/nouveau/gradlew.bat b/extra/nouveau/gradlew.bat similarity index 96% rename from nouveau/gradlew.bat rename to extra/nouveau/gradlew.bat index 25da30dbdee..7101f8e4676 100644 --- a/nouveau/gradlew.bat +++ b/extra/nouveau/gradlew.bat @@ -1,92 +1,92 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%"=="" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%"=="" set DIRNAME=. -@rem This is normally unused -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if %ERRORLEVEL% equ 0 goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if %ERRORLEVEL% equ 0 goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -set EXIT_CODE=%ERRORLEVEL% -if %EXIT_CODE% equ 0 set EXIT_CODE=1 -if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% -exit /b %EXIT_CODE% - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/nouveau/settings.gradle b/extra/nouveau/settings.gradle similarity index 100% rename from nouveau/settings.gradle rename to extra/nouveau/settings.gradle diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java similarity index 89% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index 7179eadc06a..c2230d1eb9c 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -24,10 +24,11 @@ import org.apache.couchdb.nouveau.core.UserAgentFilter; import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck; import org.apache.couchdb.nouveau.health.IndexHealthCheck; -import org.apache.couchdb.nouveau.lucene9.Lucene9Module; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.LuceneModule; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; +import org.apache.couchdb.nouveau.resources.WelcomeResource; import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask; public class NouveauApplication extends Application { @@ -65,7 +66,11 @@ public void run(NouveauApplicationConfiguration configuration, Environment envir environment.lifecycle().manage(indexManager); // Serialization classes - environment.getObjectMapper().registerModule(new Lucene9Module()); + environment.getObjectMapper().registerModule(new LuceneModule()); + + // WelcomeResource + final WelcomeResource welcomeResource = new WelcomeResource(); + environment.jersey().register(welcomeResource); // AnalyzeResource final AnalyzeResource analyzeResource = new AnalyzeResource(); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplicationConfiguration.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplicationConfiguration.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplicationConfiguration.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplicationConfiguration.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeRequest.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeRequest.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeRequest.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeRequest.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeResponse.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeResponse.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeResponse.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/AnalyzeResponse.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentDeleteRequest.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentUpdateRequest.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentUpdateRequest.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentUpdateRequest.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DocumentUpdateRequest.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleField.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleField.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleField.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleField.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleRange.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleRange.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleRange.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/DoubleRange.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Field.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Field.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/Field.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Field.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java similarity index 70% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java index 3d79fca654e..11e14082baa 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java @@ -13,15 +13,25 @@ package org.apache.couchdb.nouveau.api; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotEmpty; import java.util.Map; @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public class IndexDefinition { + public static final int LEGACY_LUCENE_VERSION = 9; + public static final int LATEST_LUCENE_VERSION = 10; + + @Min(LEGACY_LUCENE_VERSION) + @Max(LATEST_LUCENE_VERSION) + private int luceneVersion = LEGACY_LUCENE_VERSION; // Legacy version if not set. + @NotEmpty private String defaultAnalyzer; @@ -31,11 +41,27 @@ public IndexDefinition() { // Jackson deserialization } - public IndexDefinition(final String defaultAnalyzer, final Map fieldAnalyzers) { + public IndexDefinition( + final int luceneVersion, final String defaultAnalyzer, final Map fieldAnalyzers) { + this.luceneVersion = luceneVersion; this.defaultAnalyzer = defaultAnalyzer; this.fieldAnalyzers = fieldAnalyzers; } + @JsonProperty + public int getLuceneVersion() { + return luceneVersion; + } + + @JsonIgnore + public boolean isLatestVersion() { + return luceneVersion == LATEST_LUCENE_VERSION; + } + + public void setLuceneVersion(int luceneVersion) { + this.luceneVersion = luceneVersion; + } + @JsonProperty public String getDefaultAnalyzer() { return defaultAnalyzer; @@ -62,6 +88,7 @@ public boolean hasFieldAnalyzers() { public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + luceneVersion; result = prime * result + ((defaultAnalyzer == null) ? 0 : defaultAnalyzer.hashCode()); result = prime * result + ((fieldAnalyzers == null) ? 0 : fieldAnalyzers.hashCode()); return result; @@ -73,6 +100,7 @@ public boolean equals(Object obj) { if (obj == null) return false; if (getClass() != obj.getClass()) return false; IndexDefinition other = (IndexDefinition) obj; + if (luceneVersion != other.luceneVersion) return false; if (defaultAnalyzer == null) { if (other.defaultAnalyzer != null) return false; } else if (!defaultAnalyzer.equals(other.defaultAnalyzer)) return false; @@ -84,6 +112,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return "IndexDefinition [defaultAnalyzer=" + defaultAnalyzer + ", fieldAnalyzers=" + fieldAnalyzers + "]"; + return "IndexDefinition [luceneVersion=" + luceneVersion + ", defaultAnalyzer=" + defaultAnalyzer + + ", fieldAnalyzers=" + fieldAnalyzers + "]"; } } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfo.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfoRequest.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfoRequest.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfoRequest.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexInfoRequest.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Ok.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Range.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Range.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/Range.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/Range.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchHit.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchHit.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchHit.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchHit.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchRequest.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchRequest.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchRequest.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchRequest.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchResults.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchResults.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchResults.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/SearchResults.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StoredField.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StoredField.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/StoredField.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StoredField.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StringField.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StringField.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/StringField.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/StringField.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/TextField.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/TextField.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/api/TextField.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/TextField.java diff --git a/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java new file mode 100644 index 00000000000..6043da4c7ce --- /dev/null +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java @@ -0,0 +1,34 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; + +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public final class WelcomeResponse { + + public static final WelcomeResponse INSTANCE = new WelcomeResponse(); + + private final int[] supportedLuceneVersions = + new int[] {IndexDefinition.LEGACY_LUCENE_VERSION, IndexDefinition.LATEST_LUCENE_VERSION}; + + private WelcomeResponse() {} + + @JsonProperty + public int[] getSupportedLuceneVersions() { + return supportedLuceneVersions; + } +} diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IOUtils.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IOUtils.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/IOUtils.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IOUtils.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexFunction.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexFunction.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexFunction.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexFunction.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index c4f59f7a15f..4067716eed9 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -34,11 +34,12 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.couchdb.nouveau.api.IndexDefinition; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; -import org.apache.couchdb.nouveau.lucene9.Lucene9Index; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneIndex; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.misc.store.DirectIODirectory; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; @@ -218,7 +219,6 @@ public void create(final String name, IndexDefinition indexDefinition) throws IO assertSame(indexDefinition, loadIndexDefinition(name)); return; } - final Lock lock = this.createLock.writeLock(name); lock.lock(); try { @@ -392,15 +392,19 @@ private Index load(final String name) throws IOException { LOGGER.info("opening {}", name); final Path path = indexPath(name); final IndexDefinition indexDefinition = loadIndexDefinition(name); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); - final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve("9"))); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); + final int luceneVersion = indexDefinition.getLuceneVersion(); + final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve(Integer.toString(luceneVersion)))); final IndexWriterConfig config = new IndexWriterConfig(analyzer); + if (luceneVersion != IndexDefinition.LATEST_LUCENE_VERSION) { + config.setOpenMode(OpenMode.APPEND); + } config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final long updateSeq = getSeq(writer, "update_seq"); final long purgeSeq = getSeq(writer, "purge_seq"); final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); - return new Lucene9Index(analyzer, writer, updateSeq, purgeSeq, searcherManager); + return new LuceneIndex(analyzer, writer, updateSeq, purgeSeq, searcherManager); } private long getSeq(final IndexWriter writer, final String key) throws IOException { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StaleIndexException.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StaleIndexException.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/StaleIndexException.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StaleIndexException.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StripedLock.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StripedLock.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/StripedLock.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/StripedLock.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UpdatesOutOfOrderException.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UpdatesOutOfOrderException.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/UpdatesOutOfOrderException.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UpdatesOutOfOrderException.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UserAgentFilter.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UserAgentFilter.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/UserAgentFilter.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/UserAgentFilter.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/ByteArrayWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/ByteArrayWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/ByteArrayWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/ByteArrayWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/DoubleWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/DoubleWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/DoubleWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/DoubleWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/FloatWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/FloatWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/FloatWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/FloatWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/IntWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/IntWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/IntWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/IntWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/LongWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/LongWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/LongWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/LongWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/NullWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/NullWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/NullWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/NullWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/PrimitiveWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/PrimitiveWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/PrimitiveWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/PrimitiveWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/StringWrapper.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/StringWrapper.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/StringWrapper.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/core/ser/StringWrapper.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheck.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheck.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheck.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheck.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java similarity index 93% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java index 0ee8fefa2e7..7e5facb2e26 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java @@ -32,14 +32,14 @@ public IndexHealthCheck(final IndexResource indexResource) { @Override protected Result check() throws Exception { - final String name = "___test9"; + final String name = "___test"; try { indexResource.deletePath(name, null); } catch (IOException e) { // Ignored, index might not exist yet. } - indexResource.createIndex(name, new IndexDefinition("standard", null)); + indexResource.createIndex(name, new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null)); try { final DocumentUpdateRequest documentUpdateRequest = new DocumentUpdateRequest(0, 1, null, Collections.emptyList()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java index 2bd47ed9740..b95230cec7b 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -60,9 +60,9 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; -public final class Lucene9AnalyzerFactory { +public final class LuceneAnalyzerFactory { - private Lucene9AnalyzerFactory() {} + private LuceneAnalyzerFactory() {} public static Analyzer fromDefinition(final IndexDefinition indexDefinition) { final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java index dad3040f328..e62ffdb5a96 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -97,7 +97,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -public class Lucene9Index extends Index { +public class LuceneIndex extends Index { private static final Sort DEFAULT_SORT = new Sort(SortField.FIELD_SCORE, new SortField("_id", SortField.Type.STRING)); @@ -106,9 +106,9 @@ public class Lucene9Index extends Index { private final Analyzer analyzer; private final IndexWriter writer; private final SearcherManager searcherManager; - private final Lucene9IndexSchema schema; + private final LuceneIndexSchema schema; - public Lucene9Index( + public LuceneIndex( final Analyzer analyzer, final IndexWriter writer, final long updateSeq, @@ -290,8 +290,8 @@ private void collectHits(final IndexSearcher searcher, final TopDocs topDocs, fi hits.add(new SearchHit(doc.get("_id"), after, fields)); } - searchResults.setTotalHits(topDocs.totalHits.value); - searchResults.setTotalHitsRelation(topDocs.totalHits.relation); + searchResults.setTotalHits(topDocs.totalHits.value()); + searchResults.setTotalHitsRelation(topDocs.totalHits.relation()); searchResults.setHits(hits); } @@ -537,22 +537,22 @@ private Query parse(final SearchRequest request) { return result; } - private Lucene9IndexSchema initSchema(IndexWriter writer) { + private LuceneIndexSchema initSchema(IndexWriter writer) { var commitData = writer.getLiveCommitData(); if (commitData == null) { - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } for (var entry : commitData) { if (entry.getKey().equals("_schema")) { - return Lucene9IndexSchema.fromString(entry.getValue()); + return LuceneIndexSchema.fromString(entry.getValue()); } } - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } @Override public String toString() { - return "Lucene9Index [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + return "LuceneIndex [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + "]"; } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java similarity index 91% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java index 92cc5fc2e8f..ef335bf5a20 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -32,7 +32,7 @@ import org.apache.couchdb.nouveau.api.TextField; import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; -final class Lucene9IndexSchema { +final class LuceneIndexSchema { public enum Type { STRING, @@ -56,23 +56,23 @@ private static Type fromField(final Field field) { private final ConcurrentMap map; - private Lucene9IndexSchema(Map map) { + private LuceneIndexSchema(Map map) { this.map = new ConcurrentHashMap<>(map); this.map.put("_id", Type.STRING); } - public static Lucene9IndexSchema emptySchema() { - return new Lucene9IndexSchema(new HashMap()); + public static LuceneIndexSchema emptySchema() { + return new LuceneIndexSchema(new HashMap()); } - public static Lucene9IndexSchema fromString(final String schemaStr) { + public static LuceneIndexSchema fromString(final String schemaStr) { Objects.requireNonNull(schemaStr); if (schemaStr.isEmpty()) { return emptySchema(); } var map = Arrays.stream(schemaStr.split(",")) .collect(Collectors.toMap(i -> i.split(":")[0], i -> Type.valueOf(i.split(":")[1]))); - return new Lucene9IndexSchema(map); + return new LuceneIndexSchema(map); } public void update(final Collection fields) { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java similarity index 82% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java index 06102de84cb..35be8efe27f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java @@ -11,16 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.lucene.search.Query; -public class Lucene9Module extends SimpleModule { +public class LuceneModule extends SimpleModule { - public Lucene9Module() { - super("lucene9", Version.unknownVersion()); + public LuceneModule() { + super("lucene", Version.unknownVersion()); // Query addSerializer(Query.class, new QuerySerializer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java similarity index 98% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java index fc7acf5091d..d37ab3d37c5 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.util.Map; import org.apache.lucene.analysis.Analyzer; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java index 4553fa76b2f..bd9050af86f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.io.IOException; import java.util.concurrent.Executor; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java similarity index 99% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java index b0620e661dc..04a771cf52f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java index 5e1d5087193..7e42ee6dbdd 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; @@ -65,8 +65,8 @@ public void serialize(final Query query, final JsonGenerator gen, final Serializ for (final BooleanClause clause : booleanQuery.clauses()) { gen.writeStartObject(); gen.writeFieldName("query"); - serialize(clause.getQuery(), gen, provider); - gen.writeStringField("occur", clause.getOccur().name().toLowerCase()); + serialize(clause.query(), gen, provider); + gen.writeStringField("occur", clause.occur().name().toLowerCase()); gen.writeEndObject(); } gen.writeEndArray(); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java index 6b4c8c64222..464b49d6370 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java similarity index 93% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java index 2ae8b78f10a..d5249e20585 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java @@ -30,7 +30,7 @@ import java.util.List; import org.apache.couchdb.nouveau.api.AnalyzeRequest; import org.apache.couchdb.nouveau.api.AnalyzeResponse; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -47,7 +47,7 @@ public final class AnalyzeResource { public AnalyzeResponse analyzeText(@NotNull @Valid AnalyzeRequest request) throws IOException { try { final List tokens = - tokenize(Lucene9AnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); + tokenize(LuceneAnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); return new AnalyzeResponse(tokens); } catch (IllegalArgumentException e) { throw new WebApplicationException(request.getAnalyzer() + " not a valid analyzer", Status.BAD_REQUEST); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java similarity index 93% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java index a52e00da912..9ba3821090e 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/IndexResource.java @@ -26,7 +26,9 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response.Status; import java.io.IOException; import java.util.List; import java.util.Objects; @@ -57,6 +59,10 @@ public IndexResource(final IndexManager indexManager) { @PUT public Ok createIndex(@PathParam("name") String name, @NotNull @Valid IndexDefinition indexDefinition) throws IOException { + if (!indexDefinition.isLatestVersion()) { + throw new WebApplicationException( + "Cannot create a new version " + indexDefinition.getLuceneVersion() + " index", Status.BAD_REQUEST); + } indexManager.create(name, indexDefinition); return Ok.INSTANCE; } diff --git a/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java new file mode 100644 index 00000000000..0992623fc02 --- /dev/null +++ b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java @@ -0,0 +1,30 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.resources; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.apache.couchdb.nouveau.api.WelcomeResponse; + +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +public final class WelcomeResource { + + @GET + public WelcomeResponse welcome() { + return WelcomeResponse.INSTANCE; + } +} diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/CloseAllIndexesTask.java b/extra/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/CloseAllIndexesTask.java similarity index 100% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/CloseAllIndexesTask.java rename to extra/nouveau/src/main/java/org/apache/couchdb/nouveau/tasks/CloseAllIndexesTask.java diff --git a/nouveau/src/main/resources/banner.txt b/extra/nouveau/src/main/resources/banner.txt similarity index 100% rename from nouveau/src/main/resources/banner.txt rename to extra/nouveau/src/main/resources/banner.txt diff --git a/nouveau/src/main/resources/openapi.yaml b/extra/nouveau/src/main/resources/openapi.yaml similarity index 100% rename from nouveau/src/main/resources/openapi.yaml rename to extra/nouveau/src/main/resources/openapi.yaml diff --git a/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java new file mode 100644 index 00000000000..2e1e7ee0353 --- /dev/null +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java @@ -0,0 +1,42 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class IndexDefinitionTest { + + private static ObjectMapper mapper; + + @BeforeAll + public static void setup() { + mapper = new ObjectMapper(); + } + + @Test + public void legacyLuceneVersionIfMissing() throws Exception { + var indexDefinition = mapper.readValue("{}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(IndexDefinition.LEGACY_LUCENE_VERSION); + } + + @Test + public void luceneVersionIsDeserializedIfPresent() throws Exception { + var indexDefinition = mapper.readValue("{\"lucene_version\":10}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(10); + } +} diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java similarity index 100% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/api/SearchRequestTest.java diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java similarity index 83% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index bb8f0b6470e..7a122e67c85 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.SearchRequest; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -61,8 +61,8 @@ public void cleanup() throws Exception { @Test public void managerReturnsUsableIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("foo", indexDefinition); var searchRequest = new SearchRequest(); searchRequest.setQuery("*:*"); @@ -72,8 +72,8 @@ public void managerReturnsUsableIndex() throws Exception { @Test public void managerReopensAClosedIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("bar", indexDefinition); @@ -90,8 +90,8 @@ public void managerReopensAClosedIndex() throws Exception { @Test public void deleteAllRemovesIndexByName() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("bar", indexDefinition); @@ -102,8 +102,8 @@ public void deleteAllRemovesIndexByName() throws Exception { @Test public void deleteAllRemovesIndexByPath() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -114,8 +114,8 @@ public void deleteAllRemovesIndexByPath() throws Exception { @Test public void deleteAllRemovesIndexByGlob() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -126,8 +126,8 @@ public void deleteAllRemovesIndexByGlob() throws Exception { @Test public void deleteAllRemovesIndexByGlobExceptExclusions() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheckTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheckTest.java similarity index 100% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheckTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/health/AnalyzeHealthCheckTest.java diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/health/IndexHealthCheckTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/health/IndexHealthCheckTest.java similarity index 100% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/health/IndexHealthCheckTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/health/IndexHealthCheckTest.java diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java similarity index 92% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java index eb8e30e0884..383cf34dc8a 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -62,7 +62,7 @@ import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.junit.jupiter.api.Test; -public class Lucene9AnalyzerFactoryTest { +public class LuceneAnalyzerFactoryTest { @Test public void testkeyword() throws Exception { @@ -256,9 +256,11 @@ public void testturkish() throws Exception { @Test public void testFieldAnalyzers() throws Exception { - final IndexDefinition indexDefinition = - new IndexDefinition("standard", Map.of("english", "english", "thai", "thai", "email", "email")); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final IndexDefinition indexDefinition = new IndexDefinition( + IndexDefinition.LATEST_LUCENE_VERSION, + "standard", + Map.of("english", "english", "thai", "thai", "email", "email")); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); assertThat(analyzer).isInstanceOf(PerFieldAnalyzerWrapper.class); final Method m = PerFieldAnalyzerWrapper.class.getDeclaredMethod("getWrappedAnalyzer", String.class); m.setAccessible(true); @@ -270,12 +272,13 @@ public void testFieldAnalyzers() throws Exception { @Test public void testUnknownAnalyzer() throws Exception { - assertThrows(WebApplicationException.class, () -> Lucene9AnalyzerFactory.newAnalyzer("foo")); + assertThrows(WebApplicationException.class, () -> LuceneAnalyzerFactory.newAnalyzer("foo")); } private void assertAnalyzer(final String name, final Class clazz) throws Exception { - assertThat(Lucene9AnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); - assertThat(Lucene9AnalyzerFactory.fromDefinition(new IndexDefinition(name, null))) + assertThat(LuceneAnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); + assertThat(LuceneAnalyzerFactory.fromDefinition( + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, name, null))) .isInstanceOf(clazz); } } diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java similarity index 96% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java index 428e3eb6ae0..f87af2fe0f7 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -45,18 +45,18 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public class Lucene9IndexTest { +public class LuceneIndexTest { protected final Index setup(final Path path) throws IOException { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); final Directory dir = new DirectIODirectory(FSDirectory.open(path)); final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final SearcherManager searcherManager = new SearcherManager(writer, null); - return new Lucene9Index(analyzer, writer, 0L, 0L, searcherManager); + return new LuceneIndex(analyzer, writer, 0L, 0L, searcherManager); } protected final void cleanup(final Index index) throws IOException { diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java similarity index 98% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java index 0f4689b2137..ab7c3dfdd68 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java similarity index 97% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java rename to extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java index 555495a7a33..9d20907c948 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java +++ b/extra/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -39,7 +39,7 @@ public class QuerySerializationTest { @BeforeAll public static void setup() { mapper = new ObjectMapper(); - mapper.registerModule(new Lucene9Module()); + mapper.registerModule(new LuceneModule()); } @Test diff --git a/nouveau/src/test/resources/fixtures/DocumentUpdateRequest.json b/extra/nouveau/src/test/resources/fixtures/DocumentUpdateRequest.json similarity index 100% rename from nouveau/src/test/resources/fixtures/DocumentUpdateRequest.json rename to extra/nouveau/src/test/resources/fixtures/DocumentUpdateRequest.json diff --git a/nouveau/src/test/resources/fixtures/SearchRequest.json b/extra/nouveau/src/test/resources/fixtures/SearchRequest.json similarity index 100% rename from nouveau/src/test/resources/fixtures/SearchRequest.json rename to extra/nouveau/src/test/resources/fixtures/SearchRequest.json diff --git a/mise.toml b/mise.toml new file mode 100644 index 00000000000..733c7c6ea56 --- /dev/null +++ b/mise.toml @@ -0,0 +1,9 @@ +[env] +NODE_ENV = 'production' + +[tools] +elixir = "1.19.5-otp-26" +erlang = '26' +java = '21' +nodejs = '24' +python = '3' diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 762b8ecb50b..6fbc99f586c 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -38,7 +38,18 @@ view_index_dir = {{view_index_dir}} ;default_security = admin_only ;btree_chunk_size = 1279 + +; When set to "true" the node will stop processing interactive requests and +; return "status":"maintenance_mode" from the /_up endpoint. If a load balancer +; is monitoring the /_up endpoint, it can then take the node out of rotation. ;maintenance_mode = false + +; Toggle to "true" when upgrading a cluster. Then, after upgrading all nodes, +; toggle it back to "false". When this is value enabled the scanner will pause +; running any plugins. In the future other background components may pause +; their execution as well. +;upgrade_in_progress = false + ;stem_interactive_updates = true ;uri_file = @@ -167,6 +178,10 @@ view_index_dir = {{view_index_dir}} ; caching set the value to 0 ;db_btree_cache_depth = 3 +; Cache view btree nodes up to this depth only. Works like db_btree_cache_depth +; but for map-reduce (view) b-trees. +;view_btree_cache_depth = 3 + [purge] ; Allowed maximum number of accumulated revisions in one purge request ;max_revisions_number = infinity @@ -1093,6 +1108,7 @@ url = {{nouveau_url}} ;couch_scanner_plugin_find = false ;couch_scanner_plugin_conflict_finder = false ;couch_quickjs_scanner_plugin = false +;nouveau_index_upgrader = false ; The following [$plugin*] settings apply to all plugins @@ -1203,14 +1219,30 @@ url = {{nouveau_url}} ; The fewest id/rev pairs the plugin will attempt to purge in ; one request, excepting at the end of a database scan. ;min_batch_size = 250 + ; The most id/rev pairs the plugin will attempt to purge in ; one request. ;max_batch_size = 500 + ; The default time-to-live, measured in seconds, before a ; deleted document is eligible to be purged by the plugin. ; Defaults to undefined, which disables auto purging. ;deleted_document_ttl = +; Set the log level for starting, stopping and purge report summary log entries. +;log_level = info + +; When set to "true" the plugin does everything (scanning, revision processing, +; etc) but skips the purge step. Optionally use the "log_level" plugin setting +; to increase the severity of log reports so it's clear when the plugin starts, +; stops and how many revisions it found to purge. +;dry_run = false + +[nouveau_index_upgrader] +; Common scanner scheduling settings +;after = restart +;repeat = restart + [chttpd_auth_lockout] ; CouchDB can temporarily lock out IP addresses that repeatedly fail authentication ; mode can be set to one of three recognised values; @@ -1238,3 +1270,20 @@ url = {{nouveau_url}} ; periodically reload configuration from file. ; Set to infinity to disable. ;auto_reload_secs = infinity + +[hibernate_after] +; Some processes which handle a large number of referenced binaries can benefit +; from hibernating periodically, so they can run a complete garbage collection +; and dereference those binaries. This section configures idle hibernation +; timeouts for some of those processes. The value is milliseconds or +; "infinity". Setting the value to "infinity" disables hibernation. The option +; may be used as a mitigation strategy when running with earlier OTP 27/28 +; versionss which had a bug [1] which prevented processes from waking up from +; hibernation. +; +; [1] https://github.com/erlang/otp/issues/10651 + +;rexi_buffer = 5000 +;couch_stream = 5000 +;couch_db_updater = 5000 +;couch_work_queue = 5000 diff --git a/rel/plugins/eunit_plugin.erl b/rel/plugins/eunit_plugin.erl index 356f4e0f771..eeb8ebe9ea5 100644 --- a/rel/plugins/eunit_plugin.erl +++ b/rel/plugins/eunit_plugin.erl @@ -25,37 +25,56 @@ is_base_dir(RebarConf) -> filename:absname(rebar_utils:get_cwd()) =:= rebar_config:get_xconf(RebarConf, base_dir, undefined). -build_eunit_config(Config0, AppFile) -> +build_eunit_config(Config, AppFile) -> Cwd = filename:absname(rebar_utils:get_cwd()), - DataDir = Cwd ++ "/tmp/data", - ViewIndexDir = Cwd ++ "/tmp/data", - StateDir = Cwd ++ "/tmp/data", - TmpDataDir = Cwd ++ "/tmp/tmp_data", - LogDir = Cwd ++ "/tmp", - cleanup_dirs([DataDir, TmpDataDir]), - Config1 = rebar_config:set_global(Config0, template, "setup_eunit"), - Config2 = rebar_config:set_global(Config1, prefix, Cwd), + App = rebar_config:get_global(Config, app, undefined), + case is_list(App) of + true -> ok; + false -> error(app_parameter_must_be_defined) + end, + case re:run(App, "^[_a-z0-9]+$") of + nomatch -> + error({app_parameter_must_be_just_one_app, App}); + {match, _} -> + Prefix = Cwd ++ "/tmp/" ++ App, + DataDir = Prefix ++ "/data", + ViewIndexDir = Prefix ++ "/data", + StateDir = Prefix ++ "/data", + TmpDataDir = Prefix ++ "/tmp_data", + EtcDir = Prefix ++ "/etc", + LogDir = Prefix, + build_config( + Config, + AppFile, + Prefix, + DataDir, + ViewIndexDir, + StateDir, + TmpDataDir, + EtcDir, + LogDir + ) + end. + +build_config(Config, AppFile, Prefix, DataDir, ViewIndexDir, StateDir, TmpDataDir, EtcDir, LogDir) -> + cleanup_dirs([DataDir, TmpDataDir, EtcDir]), + Config1 = rebar_config:set_global(Config, template, "setup_eunit"), + Config2 = rebar_config:set_global(Config1, prefix, Prefix), Config3 = rebar_config:set_global(Config2, data_dir, DataDir), Config4 = rebar_config:set_global(Config3, view_index_dir, ViewIndexDir), Config5 = rebar_config:set_global(Config4, log_dir, LogDir), - Config = rebar_config:set_global(Config5, state_dir, StateDir), - rebar_templater:create(Config, AppFile). + Config6 = rebar_config:set_global(Config5, etc_dir, EtcDir), + Config7 = rebar_config:set_global(Config6, tmp_data, TmpDataDir), + Config8 = rebar_config:set_global(Config7, state_dir, StateDir), + rebar_templater:create(Config8, AppFile). cleanup_dirs(Dirs) -> lists:foreach( fun(Dir) -> case filelib:is_dir(Dir) of - true -> del_dir(Dir); + true -> file:del_dir_r(Dir); false -> ok end end, Dirs ). - -del_dir(Dir) -> - All = filelib:wildcard(Dir ++ "/**"), - {Dirs, Files} = lists:partition(fun filelib:is_dir/1, All), - ok = lists:foreach(fun file:delete/1, Files), - SortedDirs = lists:sort(fun(A, B) -> length(A) > length(B) end, Dirs), - ok = lists:foreach(fun file:del_dir/1, SortedDirs), - ok = file:del_dir(Dir). diff --git a/setup_eunit.template b/setup_eunit.template index 11eee4458a1..08705b3bf20 100644 --- a/setup_eunit.template +++ b/setup_eunit.template @@ -1,21 +1,24 @@ {variables, [ {package_author_name, "The Apache Software Foundation"}, - {cluster_port, 5984}, - {backend_port, 5986}, - {prometheus_port, 17986}, + {cluster_port, 0}, + {backend_port, 0}, + {prometheus_port, 0}, {node_name, "-name couchdbtest@127.0.0.1"}, {data_dir, "/tmp"}, {prefix, "/tmp"}, {view_index_dir, "/tmp"}, {state_dir, "/tmp"}, - {log_dir, "/tmp"} + {log_dir, "/tmp"}, + {etc_dir, "/tmp"}, + {tmp_data, "/tmp"} ]}. -{dir, "tmp"}. -{dir, "tmp/etc"}. -{dir, "tmp/data"}. -{dir, "tmp/tmp_data"}. -{template, "rel/overlay/etc/default.ini", "tmp/etc/default_eunit.ini"}. -{template, "rel/overlay/etc/local.ini", "tmp/etc/local_eunit.ini"}. -{template, "rel/files/eunit.ini", "tmp/etc/eunit.ini"}. -{template, "rel/overlay/etc/vm.args", "tmp/etc/vm.args"}. +{dir, "{{prefix}}"}. +{dir, "{{etc_dir}}"}. +{dir, "{{data_dir}}"}. +{dir, "{{tmp_data}}"}. + +{template, "rel/overlay/etc/default.ini", "{{prefix}}/etc/default_eunit.ini"}. +{template, "rel/overlay/etc/local.ini", "{{prefix}}/etc/local_eunit.ini"}. +{template, "rel/files/eunit.ini", "{{prefix}}/etc/eunit.ini"}. +{template, "rel/overlay/etc/vm.args", "{{prefix}}/etc/vm.args"}. diff --git a/src/b64url/README.md b/src/b64url/README.md index c3a3497f2d7..f93b31a8d8b 100644 --- a/src/b64url/README.md +++ b/src/b64url/README.md @@ -14,24 +14,45 @@ decoding Base64 URL values: ## Performance -This implementation is significantly faster than the Erlang version it replaced -in CouchDB. The `benchmark.escript` file contains the original implementation -(using regular expressions to replace unsafe characters in the output of the -`base64` module) and can be used to compare the two for strings of various -lengths. For example: +This implementation is faster than the Erlang version in OTP 26-28, +especially for larger binaries (1000+ bytes). To benchmark clone +erlperf repo and run `./benchmark.sh` script. In the future, it's +plausible Erlang OTP's base64 module may become faster than the NIF, +due to improvements in the JIT capabilities but it's not there yet. ``` -ERL_LIBS=_build/default/lib/b64url/ ./test/benchmark.escript 4 10 100 30 -erl : 75491270 bytes / 30 seconds = 2516375.67 bps -nif : 672299342 bytes / 30 seconds = 22409978.07 bps -``` +./benchmark.sh + +[...] + +--- bytes: 100 ----- +Code || QPS Time Rel +encode_otp_100 1 1613 Ki 620 ns 100% +encode_nif_100 1 1391 Ki 719 ns 86% +Code || QPS Time Rel +decode_nif_100 1 1453 Ki 688 ns 100% +decode_otp_100 1 1395 Ki 716 ns 96% + +[...] -This test invocation spawns four workers that generate random strings between 10 -and 100 bytes in length and then perform an encode/decode on them in a tight -loop for 30 seconds, and then reports the aggregate encoded data volume. Note -that the generator overhead (`crypto:strong_rand_bytes/1`) is included in these -results, so the relative difference in encoder throughput is rather larger than -what's reported here. +--- bytes: 1000 ----- +Code || QPS Time Rel +encode_nif_1000 1 369 Ki 2711 ns 100% +encode_otp_1000 1 204 Ki 4904 ns 55% +Code || QPS Time Rel +decode_nif_1000 1 455 Ki 2196 ns 100% +decode_otp_1000 1 178 Ki 5612 ns 39% + +[...] + +--- bytes: 10000000 ----- +Code || QPS Time Rel +encode_nif_10000000 1 45 22388 us 100% +encode_otp_10000000 1 19 51724 us 43% +Code || QPS Time Rel +decode_nif_10000000 1 55 18078 us 100% +decode_otp_10000000 1 17 60020 us 30% +``` ## Timeslice Consumption diff --git a/src/b64url/benchmark.sh b/src/b64url/benchmark.sh new file mode 100755 index 00000000000..d7760ed2bbf --- /dev/null +++ b/src/b64url/benchmark.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Expects erlperf to be installed +# +# $ git clone https://github.com/max-au/erlperf.git +# $ cd erlperf +# $ rebar3 as prod escriptize +# $ cd .. + +for i in 50 100 150 200 500 1000 5000 10000 50000 1000000 10000000; do + echo "" + echo "--- bytes: ${i} -----" + ERL_LIBS="." erlperf/erlperf -w 2 \ + 'runner(Bin) -> b64url:encode(Bin).' --label "encode_nif_${i}" \ + 'runner(Bin) -> base64:encode(Bin, #{mode => urlsafe, padding => false}).' --label "encode_otp_${i}" \ + --init_runner_all "rand:seed(default,{1,2,3}), rand:bytes(${i})." + + ERL_LIBS="." erlperf/erlperf -w 2 \ + 'runner(Enc) -> b64url:decode(Enc).' --label "decode_nif_${i}" \ + 'runner(Enc) -> base64:decode(Enc, #{mode => urlsafe, padding => false}).' --label "decode_otp_${i}" \ + --init_runner_all "rand:seed(default,{1,2,3}), b64url:encode(rand:bytes(round(${i} * (3/4))))." +done diff --git a/src/b64url/test/benchmark.escript b/src/b64url/test/benchmark.escript deleted file mode 100755 index 00a6f0dda3b..00000000000 --- a/src/b64url/test/benchmark.escript +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env escript - --mode(compile). - - --export([ - encode/1, - decode/1, - run_worker/1 -]). - - --record(st, { - parent, - module, - workers, - minsize, - maxsize, - duration, - total_bytes -}). - - -main([Workers0, MinSize0, MaxSize0, Duration0]) -> - code:add_path("./ebin"), - code:add_path("../ebin"), - Workers = to_int(Workers0), - MinSize = to_int(MinSize0), - MaxSize = to_int(MaxSize0), - Duration = to_int(Duration0), - if Workers > 0 -> ok; true -> - die("Worker count must be positive~n") - end, - if MinSize > 0 -> ok; true -> - die("Minimum size must be positive.~n") - end, - if MaxSize > 0 -> ok; true -> - die("Maximum size must be positive.~n") - end, - if MinSize < MaxSize -> ok; true -> - die("Minimum size must be less than maximum size.~n") - end, - if Duration > 0 -> ok; true -> - die("Duration must be positive.~n") - end, - St = #st{ - parent = self(), - workers = Workers, - minsize = MinSize, - maxsize = MaxSize, - duration = Duration - }, - lists:foreach(fun(M) -> - run_test(St#st{module=M}) - end, randomize([b64url, ?MODULE])); - -main(_) -> - Args = [escript:script_name()], - die("usage: ~s num_workers min_size max_size time_per_test~n", Args). - - -run_test(St) -> - Workers = spawn_workers(St#st.workers, St), - start_workers(Workers), - Results = wait_for_workers(Workers), - report(St#st.module, St#st.duration, Results). - - -start_workers(Pids) -> - lists:foreach(fun(P) -> - P ! start - end, Pids). - - -wait_for_workers(Pids) -> - lists:map(fun(P) -> - receive - {P, TotalBytes} -> TotalBytes - end - end, Pids). - - -report(Module, Duration, TotalByteList) -> - ModDesc = case Module of - ?MODULE -> "erl"; - b64url -> "nif" - end, - TotalBytes = lists:sum(TotalByteList), - io:format("~s : ~14b bytes / ~3b seconds = ~14.2f bps~n", [ - ModDesc, TotalBytes, Duration, TotalBytes / Duration]). - - -spawn_workers(NumWorkers, St) -> - lists:map(fun(_) -> - spawn_link(?MODULE, run_worker, [St]) - end, lists:seq(1, NumWorkers)). - - -run_worker(St) -> - receive - start -> ok - end, - run_worker(St#st{total_bytes=0}, os:timestamp()). - - -run_worker(St, Started) -> - HasRun = timer:now_diff(os:timestamp(), Started), - case HasRun div 1000000 > St#st.duration of - true -> - St#st.parent ! {self(), St#st.total_bytes}; - false -> - NewSt = do_round_trip(St), - run_worker(NewSt, Started) - end. - - -do_round_trip(St) -> - Size = St#st.minsize + rand:uniform(St#st.maxsize - St#st.minsize), - Data = crypto:strong_rand_bytes(Size), - Encoded = (St#st.module):encode(Data), - Data = (St#st.module):decode(Encoded), - St#st{total_bytes=St#st.total_bytes+Size}. - - -encode(Url) -> - Url1 = iolist_to_binary(re:replace(base64:encode(Url), "=+$", "")), - Url2 = iolist_to_binary(re:replace(Url1, "/", "_", [global])), - iolist_to_binary(re:replace(Url2, "\\+", "-", [global])). - - -decode(Url64) -> - Url1 = re:replace(iolist_to_binary(Url64), "-", "+", [global]), - Url2 = iolist_to_binary( - re:replace(iolist_to_binary(Url1), "_", "/", [global]) - ), - Padding = list_to_binary(lists:duplicate((4 - size(Url2) rem 4) rem 4, $=)), - base64:decode(<>). - -randomize(List) -> - List0 = [{rand:uniform(), L} || L <- List], - List1 = lists:sort(List0), - [L || {_, L} <- List1]. - - -to_int(Val) when is_integer(Val) -> - Val; -to_int(Val) when is_binary(Val) -> - to_int(binary_to_list(Val)); -to_int(Val) when is_list(Val) -> - try - list_to_integer(Val) - catch _:_ -> - die("Invalid integer: ~w~n", [Val]) - end; -to_int(Val) -> - die("Invalid integer: ~w~n", [Val]). - - -die(Message) -> - die(Message, []). - -die(Format, Args) -> - io:format(Format, Args), - init:stop(). - diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index c22a27b9823..761b01a1c3c 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -112,9 +112,11 @@ handle_utils_dir_req(#httpd{method = 'GET'} = Req, DocumentRoot) -> handle_utils_dir_req(Req, _) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_all_dbs_req(#httpd{method = 'GET'} = Req) -> +handle_all_dbs_req(#httpd{method = 'GET', path_parts = [<<"_all_dbs">>]} = Req) -> handle_all_dbs_info_req(Req); -handle_all_dbs_req(Req) -> +handle_all_dbs_req(#httpd{method = 'GET', path_parts = [<<"_all_dbs">> | _]} = Req) -> + chttpd:send_error(Req, not_found); +handle_all_dbs_req(#httpd{path_parts = [<<"_all_dbs">>]} = Req) -> send_method_not_allowed(Req, "GET,HEAD"). handle_all_dbs_info_req(Req) -> @@ -170,9 +172,9 @@ all_dbs_info_callback({error, Reason}, #vacc{resp = Resp0} = Acc) -> {ok, Resp1} = chttpd:send_delayed_error(Resp0, Reason), {ok, Acc#vacc{resp = Resp1}}. -handle_dbs_info_req(#httpd{method = 'GET'} = Req) -> +handle_dbs_info_req(#httpd{method = 'GET', path_parts = [<<"_dbs_info">>]} = Req) -> handle_all_dbs_info_req(Req); -handle_dbs_info_req(#httpd{method = 'POST'} = Req) -> +handle_dbs_info_req(#httpd{method = 'POST', path_parts = [<<"_dbs_info">>]} = Req) -> chttpd:validate_ctype(Req, "application/json"), Props = chttpd:json_body_obj(Req), Keys = couch_mrview_util:get_view_keys(Props), @@ -209,6 +211,10 @@ handle_dbs_info_req(#httpd{method = 'POST'} = Req) -> ), send_chunk(Resp, "]"), chttpd:end_json_response(Resp); +handle_dbs_info_req(#httpd{method = Method, path_parts = [<<"_dbs_info">> | _]} = Req) when + Method == 'GET'; Method == 'POST' +-> + chttpd:send_error(Req, not_found); handle_dbs_info_req(Req) -> send_method_not_allowed(Req, "GET,HEAD,POST"). diff --git a/src/chttpd/src/chttpd_test_util.erl b/src/chttpd/src/chttpd_test_util.erl index 8a849acda1c..43bb72cf682 100644 --- a/src/chttpd/src/chttpd_test_util.erl +++ b/src/chttpd/src/chttpd_test_util.erl @@ -17,7 +17,7 @@ -include_lib("couch/include/couch_eunit.hrl"). start_couch() -> - start_couch(?CONFIG_CHAIN). + test_util:start_couch([chttpd]). start_couch(IniFiles) -> test_util:start_couch(IniFiles, [chttpd]). diff --git a/src/config/src/config.erl b/src/config/src/config.erl index e00746208a5..eb7316672eb 100644 --- a/src/config/src/config.erl +++ b/src/config/src/config.erl @@ -87,6 +87,10 @@ get_integer(Section, Key, Default) when is_integer(Default) -> get_integer_or_infinity(Section, Key, Default) when is_integer(Default); Default == infinity -> case get_value(Section, Key, Default) of infinity -> + % We got the default infinity + infinity; + "infinity" -> + % Value was actually set to "infinity" infinity; _Value -> get_integer_int(Section, Key, Default) diff --git a/src/config/test/config_tests.erl b/src/config/test/config_tests.erl index 4e3bf60f71a..301fbf7a6a8 100644 --- a/src/config/test/config_tests.erl +++ b/src/config/test/config_tests.erl @@ -24,7 +24,7 @@ -define(RESTART_TIMEOUT_IN_MILLISEC, 3000). -define(CONFIG_FIXTURESDIR, - filename:join([?BUILDDIR(), "src", "config", "test", "fixtures"]) + ?ABS_PATH("test/fixtures") ). -define(CONFIG_DEFAULT_TEST, diff --git a/src/couch/include/couch_eunit.hrl b/src/couch/include/couch_eunit.hrl index 2c10e257de6..5bae5b452a2 100644 --- a/src/couch/include/couch_eunit.hrl +++ b/src/couch/include/couch_eunit.hrl @@ -21,19 +21,17 @@ false -> throw("BUILDDIR environment variable must be set"); Dir -> - Dir + string:strip(Dir) end end). -define(CONFIG_DEFAULT, - filename:join([?BUILDDIR(), "tmp", "etc", "default_eunit.ini"])). + filename:join([?BUILDDIR(), "etc", "default_eunit.ini"])). -define(CONFIG_CHAIN, [ ?CONFIG_DEFAULT, - filename:join([?BUILDDIR(), "tmp", "etc", "local_eunit.ini"]), - filename:join([?BUILDDIR(), "tmp", "etc", "eunit.ini"])]). --define(FIXTURESDIR, - filename:join([?BUILDDIR(), "src", "couch", "test", "eunit", "fixtures"])). + filename:join([?BUILDDIR(), "etc", "local_eunit.ini"]), + filename:join([?BUILDDIR(), "etc", "eunit.ini"])]). -define(TEMPDIR, - filename:join([?BUILDDIR(), "tmp", "tmp_data"])). + filename:join([?BUILDDIR(), "tmp_data"])). -define(APPDIR, filename:dirname(element(2, file:get_cwd()))). %% Account for the fact that source files are in src//.eunit/.erl @@ -41,6 +39,9 @@ -define(ABS_PATH(File), %% src//.eunit/.erl filename:join([?APPDIR, File])). +-define(FIXTURESDIR, + ?ABS_PATH("test/eunit/fixtures")). + -define(tempfile, fun() -> Suffix = couch_uuids:random(), diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 18f32a47a88..e3bd5299259 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -486,3 +486,11 @@ {type, counter}, {desc, <<"number of times bt_engine cache was full">>} ]}. +{[couchdb, maintenance_mode], [ + {type, gauge}, + {desc, <<"set to 1 when node is in maintenance mode">>} +]}. +{[couchdb, upgrade_in_progress], [ + {type, gauge}, + {desc, <<"set to 1 when upgrade_in_progress is toggled">>} +]}. diff --git a/src/couch/rebar.config.script b/src/couch/rebar.config.script index 1c9809902c3..f64f2268b17 100644 --- a/src/couch/rebar.config.script +++ b/src/couch/rebar.config.script @@ -98,12 +98,12 @@ SMVsn = case lists:keyfind(spidermonkey_version, 1, CouchConfig) of {_, "128"} -> "128"; undefined -> - "91"; + "128"; {_, Unsupported} -> io:format(standard_error, "Unsupported SpiderMonkey version: ~s~n", [Unsupported]), erlang:halt(1); false -> - "91" + "128" end. ConfigH = [ @@ -256,7 +256,7 @@ CouchJSEnv = case SMVsn of ] end. -BrewIcuPrefixCmd = "brew --prefix icu4c". +BrewIcuPrefixCmd = "brew --prefix icu4c 2> /dev/null". GenericIcuIncludePaths = "-I/usr/local/opt/icu4c/include -I/opt/homebrew/opt/icu4c/include". GenericIcuLibPaths = "-L/usr/local/opt/icu4c/lib -L/opt/homebrew/opt/icu4c/lib". diff --git a/src/couch/src/couch_auto_purge_plugin.erl b/src/couch/src/couch_auto_purge_plugin.erl index 34234377efd..4fd904fdd8e 100644 --- a/src/couch/src/couch_auto_purge_plugin.erl +++ b/src/couch/src/couch_auto_purge_plugin.erl @@ -28,21 +28,39 @@ -include_lib("stdlib/include/assert.hrl"). start(ScanId, #{}) -> - St = init_config(ScanId), - ?INFO("Starting.", [], St), - {ok, St}. + case dead_nodes() of + true -> + ?LOG(level(), "Not starting. Found dead nodes", [], #{sid => ScanId}), + skip; + false -> + St = init_config(ScanId), + ?LOG(level(), "Starting.", [], St), + {ok, St} + end. resume(ScanId, #{}) -> - St = init_config(ScanId), - ?INFO("Resuming.", [], St), - {ok, St}. + case dead_nodes() of + true -> + ?LOG(level(), "Not resuming. Found dead nodes", [], #{sid => ScanId}), + skip; + false -> + St = init_config(ScanId), + ?LOG(level(), "Resuming.", [], St), + {ok, St} + end. complete(St) -> - ?INFO("Completed", [], St), + ?LOG(level(), "Completed.", [], St), {ok, #{}}. -checkpoint(_St) -> - {ok, #{}}. +checkpoint(St) -> + case dead_nodes() of + true -> + ?WARN("Stopping. Found dead nodes", [], meta(St)), + {stop, #{}}; + false -> + {ok, #{}} + end. db(St, DbName) -> case ttl(St, DbName) of @@ -66,7 +84,13 @@ db_opened(#{} = St, Db) -> db_closing(#{} = St, Db) -> St1 = #{count := Count} = flush_queue(St, Db), - ?INFO("purged ~B deleted documents from ~s", [Count, couch_db:name(Db)], meta(St1)), + case Count > 0 of + true -> + LogArgs = [Count, couch_db:name(Db)], + ?LOG(level(), "purged ~B deleted documents from ~s", LogArgs, meta(St1)); + false -> + ok + end, {ok, St1}. doc_fdi(#{} = St, #full_doc_info{deleted = true} = FDI, Db) -> @@ -103,7 +127,11 @@ flush_queue(#{queue := []} = St, _Db) -> flush_queue(#{queue := IdRevs} = St, Db) -> DbName = mem3:dbname(couch_db:name(Db)), N = mem3:n(DbName), - PurgeFun = fun() -> fabric:purge_docs(DbName, IdRevs, [?ADMIN_CTX, {w, N}]) end, + PurgeFun = + case dry_run() of + false -> fun() -> fabric:purge_docs(DbName, IdRevs, [?ADMIN_CTX, {w, N}]) end; + true -> fun() -> {ok, [{ok, Revs} || {_Id, Revs} <- IdRevs]} end + end, Timeout = fabric_util:request_timeout(), try fabric_util:isolate(PurgeFun, Timeout) of {Health, Results} when Health == ok; Health == accepted -> @@ -213,3 +241,13 @@ min_batch_size() -> max_batch_size() -> erlang:max(min_batch_size(), config:get_integer(atom_to_list(?MODULE), "max_batch_size", 500)). + +dry_run() -> + config:get_boolean(atom_to_list(?MODULE), "dry_run", false). + +level() -> + Level = config:get(atom_to_list(?MODULE), "log_level", "info"), + couch_log_util:level_to_atom(Level). + +dead_nodes() -> + [] =/= (mem3:nodes() -- mem3_util:live_nodes()). diff --git a/src/couch/src/couch_bt_engine_cache.erl b/src/couch/src/couch_bt_engine_cache.erl index a3956495772..0b3d027044a 100644 --- a/src/couch/src/couch_bt_engine_cache.erl +++ b/src/couch/src/couch_bt_engine_cache.erl @@ -13,6 +13,7 @@ -module(couch_bt_engine_cache). -include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("couch/include/couch_db.hrl"). % Main API % @@ -52,7 +53,7 @@ -define(MISSES, misses). -define(FULL, full). --record(cache, {tid, max_size}). +-record(cache, {tid, max_size, max_term}). % Main API @@ -60,11 +61,11 @@ insert(Key, Term) -> insert(Key, Term, 1). insert(Key, Term, Priority) when is_integer(Priority) -> - Priority1 = min(?MAX_PRIORITY, max(0, Priority)), case get_cache(Key) of - #cache{tid = Tid, max_size = Max} -> - case ets:info(Tid, memory) < Max of + #cache{tid = Tid, max_size = Max, max_term = MaxTerm} -> + case ets:info(Tid, memory) < Max andalso ?term_size(Term) =< MaxTerm of true -> + Priority1 = min(?MAX_PRIORITY, max(0, Priority)), case ets:insert_new(Tid, {Key, Priority1, Term}) of true -> true; @@ -107,11 +108,13 @@ info() -> Caches when is_tuple(Caches) -> SizeMem = [info(C) || C <- tuple_to_list(Caches)], MaxMem = [Max || #cache{max_size = Max} <- tuple_to_list(Caches)], + #cache{max_term = MaxTerm} = element(1, Caches), {Sizes, Mem} = lists:unzip(SizeMem), #{ size => lists:sum(Sizes), memory => lists:sum(Mem), max_memory => lists:sum(MaxMem) * wordsize(), + max_term => MaxTerm, full => sample_metric(?FULL), hits => sample_metric(?HITS), misses => sample_metric(?MISSES), @@ -207,8 +210,13 @@ new() -> Opts = [public, {write_concurrency, true}, {read_concurrency, true}], Max0 = round(max_size() / wordsize() / shard_count()), % Some per-table overhead for the table metadata - Max = Max0 + round(250 * 1024 / wordsize()), - #cache{tid = ets:new(?MODULE, Opts), max_size = Max}. + MaxSize = Max0 + round(250 * 1024 / wordsize()), + MaxTerm = couch_btree:get_chunk_size() * 4, + #cache{ + tid = ets:new(?MODULE, Opts), + max_size = MaxSize, + max_term = MaxTerm + }. get_cache(Term) -> case persistent_term:get(?PTERM_KEY, undefined) of diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 85d33cf9512..2414285b4e9 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -163,7 +163,20 @@ copy_purge_info(#comp_st{} = CompSt) -> % stale or deprecated internal replicator checkpoints beforehand. ok = mem3_rep:cleanup_purge_checkpoints(DbName), MinPurgeSeq = couch_util:with_db(DbName, fun(Db) -> - couch_db:get_minimum_purge_seq(Db) + % If we don't (yet) have all the expected internal replicator purge + % checkpoints, use the oldest purge sequence instead of the minimum. + % This is to avoid the removing some purge infos too early before the + % checkpoint is created. For example, if the oldest sequence = 1, + % minimum sequence = 1000, and current purge sequence = 2000, we can + % compact and remove all the purge infos from 1 to 1000. While + % compaction happens, a checkpoint is created with sequence = 500. In + % that case we'd end up with a "hole" between 500 and 1001 -- a new + % minimum purge sequence of 500, but the oldest checkpoint is would be + % 1001. + case mem3_rep:have_all_purge_checkpoints(Db) of + true -> couch_db:get_minimum_purge_seq(Db); + false -> couch_db:get_oldest_purge_seq(Db) + end end), OldPSTree = OldSt#st.purge_seq_tree, StartSeq = couch_bt_engine:get_purge_seq(NewSt) + 1, diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl index 1519b1fbcbd..1c349d00248 100644 --- a/src/couch/src/couch_btree.erl +++ b/src/couch/src/couch_btree.erl @@ -17,6 +17,7 @@ -export([fold_reduce/4, lookup/2, set_options/2]). -export([is_btree/1, get_state/1, get_fd/1, get_reduce_fun/1]). -export([extract/2, assemble/3, less/3]). +-export([get_chunk_size/0]). -include_lib("couch/include/couch_db.hrl"). diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 8394cd5a3ca..0d89c47507d 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -49,7 +49,8 @@ init({Engine, DbName, FilePath, Options0}) -> % couch_db:validate_doc_update, which loads them lazily. NewDb = Db#db{main_pid = self()}, proc_lib:init_ack({ok, NewDb}), - gen_server:enter_loop(?MODULE, [], NewDb) + GenOpts = couch_util:hibernate_after(?MODULE), + gen_server:enter_loop(?MODULE, GenOpts, NewDb) catch throw:InitError -> proc_lib:init_ack(InitError) @@ -220,11 +221,11 @@ handle_info( false -> Db2 end, - {noreply, Db3, hibernate} + {noreply, Db3} catch throw:retry -> [catch (ClientPid ! {retry, self()}) || ClientPid <- Clients], - {noreply, Db, hibernate} + {noreply, Db} end; handle_info({'EXIT', _Pid, normal}, Db) -> {noreply, Db}; diff --git a/src/couch/src/couch_httpd_auth.erl b/src/couch/src/couch_httpd_auth.erl index 7c6a60d2b9f..3779be66bcf 100644 --- a/src/couch/src/couch_httpd_auth.erl +++ b/src/couch/src/couch_httpd_auth.erl @@ -209,14 +209,15 @@ proxy_auth_user(Req) -> case chttpd_util:get_chttpd_auth_config("secret") of undefined -> Req#httpd{user_ctx = #user_ctx{name = ?l2b(UserName), roles = Roles}}; - Secret -> - HashAlgorithms = couch_util:get_config_hash_algorithms(), - Token = header_value(Req, XHeaderToken), - VerifyTokens = fun(HashAlg) -> - Hmac = couch_util:hmac(HashAlg, Secret, UserName), - couch_passwords:verify(couch_util:to_hex(Hmac), Token) - end, - case lists:any(VerifyTokens, HashAlgorithms) of + _Secret -> + Token = + try + binary:decode_hex(?l2b(header_value(Req, XHeaderToken))) + catch + error:badarg -> + undefined + end, + case couch_secrets:verify(UserName, Token) of true -> Req#httpd{ user_ctx = #user_ctx{ @@ -355,35 +356,30 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) -> end, % Verify expiry and hash CurrentTime = make_cookie_time(), - HashAlgorithms = couch_util:get_config_hash_algorithms(), case chttpd_util:get_chttpd_auth_config("secret") of undefined -> couch_log:debug("cookie auth secret is not set", []), Req; - SecretStr -> - Secret = ?l2b(SecretStr), + _SecretStr -> case AuthModule:get_user_creds(Req, User) of nil -> Req; {ok, UserProps, _AuthCtx} -> UserSalt = couch_util:get_value(<<"salt">>, UserProps, <<"">>), - FullSecret = <>, Hash = ?l2b(HashStr), - VerifyHash = fun(HashAlg) -> - Hmac = couch_util:hmac( - HashAlg, - FullSecret, - lists:join(":", [User, TimeStr]) - ), - couch_passwords:verify(Hmac, Hash) - end, Timeout = chttpd_util:get_chttpd_auth_config_integer( "timeout", 600 ), couch_log:debug("timeout ~p", [Timeout]), case (catch list_to_integer(TimeStr, 16)) of TimeStamp when CurrentTime < TimeStamp + Timeout -> - case lists:any(VerifyHash, HashAlgorithms) of + case + couch_secrets:verify( + lists:join(":", [User, TimeStr]), + UserSalt, + Hash + ) + of true -> TimeLeft = TimeStamp + Timeout - CurrentTime, couch_log:debug( @@ -398,7 +394,7 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) -> ) }, auth = - {FullSecret, TimeLeft < Timeout * 0.9} + {UserSalt, TimeLeft < Timeout * 0.9} }; _Else -> Req @@ -413,7 +409,7 @@ cookie_authentication_handler(#httpd{mochi_req = MochiReq} = Req, AuthModule) -> cookie_auth_header(#httpd{user_ctx = #user_ctx{name = null}}, _Headers) -> []; cookie_auth_header( - #httpd{user_ctx = #user_ctx{name = User}, auth = {Secret, _SendCookie = true}} = + #httpd{user_ctx = #user_ctx{name = User}, auth = {UserSalt, _SendCookie = true}} = Req, Headers ) -> @@ -430,21 +426,20 @@ cookie_auth_header( if AuthSession == undefined -> TimeStamp = make_cookie_time(), - [cookie_auth_cookie(Req, User, Secret, TimeStamp)]; + [cookie_auth_cookie(Req, User, UserSalt, TimeStamp)]; true -> [] end; cookie_auth_header(_Req, _Headers) -> []. -cookie_auth_cookie(Req, User, Secret, TimeStamp) -> +cookie_auth_cookie(Req, User, UserSalt, TimeStamp) -> SessionItems = [User, integer_to_list(TimeStamp, 16)], - cookie_auth_cookie(Req, Secret, SessionItems). + cookie_auth_cookie(Req, UserSalt, SessionItems). -cookie_auth_cookie(Req, Secret, SessionItems) when is_list(SessionItems) -> +cookie_auth_cookie(Req, UserSalt, SessionItems) when is_list(SessionItems) -> SessionData = lists:join(":", SessionItems), - [HashAlgorithm | _] = couch_util:get_config_hash_algorithms(), - Hash = couch_util:hmac(HashAlgorithm, Secret, SessionData), + Hash = couch_secrets:sign(SessionData, UserSalt), mochiweb_cookies:cookie( "AuthSession", couch_util:encodeBase64Url(lists:join(":", [SessionData, Hash])), @@ -514,11 +509,11 @@ handle_session_req(#httpd{method = 'POST', mochi_req = MochiReq} = Req, AuthModu Req, UserName, Password, UserProps, AuthModule, AuthCtx ), % setup the session cookie - Secret = ?l2b(ensure_cookie_auth_secret()), + ensure_cookie_auth_secret(), UserSalt = couch_util:get_value(<<"salt">>, UserProps), CurrentTime = make_cookie_time(), Cookie = cookie_auth_cookie( - Req, UserName, <>, CurrentTime + Req, UserName, UserSalt, CurrentTime ), % TODO document the "next" feature in Futon {Code, Headers} = diff --git a/src/couch/src/couch_password_hasher.erl b/src/couch/src/couch_password_hasher.erl index 677d1c2f596..6f9f3dc15a6 100644 --- a/src/couch/src/couch_password_hasher.erl +++ b/src/couch/src/couch_password_hasher.erl @@ -122,7 +122,7 @@ needs_upgrade(UserProps) -> "iterations", 600000 ), case {TargetScheme, TargetIterations, TargetPRF} of - {CurrentScheme, CurrentIterations, _} when CurrentScheme == <<"simple">> -> + {CurrentScheme, _, _} when CurrentScheme == <<"simple">> -> false; {CurrentScheme, CurrentIterations, CurrentPRF} when CurrentScheme == <<"pbkdf2">> -> false; diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl index 8fc3c9a13d7..766235d5d03 100644 --- a/src/couch/src/couch_secondary_sup.erl +++ b/src/couch/src/couch_secondary_sup.erl @@ -24,6 +24,7 @@ init([]) -> ], Daemons = [ + {couch_secrets, {couch_secrets, start_link, []}}, {query_servers, {couch_proc_manager, start_link, []}}, {vhosts, {couch_httpd_vhost, start_link, []}}, {uuids, {couch_uuids, start, []}}, diff --git a/src/couch/src/couch_secrets.erl b/src/couch/src/couch_secrets.erl new file mode 100644 index 00000000000..46269ecbc00 --- /dev/null +++ b/src/couch/src/couch_secrets.erl @@ -0,0 +1,213 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_secrets). + +-behaviour(gen_server). +-behaviour(config_listener). + +-include_lib("couch/include/couch_db.hrl"). + +%% public api +-export([sign/1, sign/2, verify/2, verify/3, secret_is_set/0]). + +%% gen_server functions +-export([ + start_link/0, + init/1, + handle_call/3, + handle_cast/2, + handle_continue/2, + handle_info/2 +]). + +%% config_listener functions +-export([ + handle_config_change/5, + handle_config_terminate/3 +]). + +sign(Message) -> + sign(Message, <<>>). + +sign(Message, ExtraSecret) -> + [HashAlgorithm | _] = couch_util:get_config_hash_algorithms(), + case current_secret() of + undefined -> + throw({internal_server_error, <<"cookie auth secret is not set">>}); + CurrentSecret -> + FullSecret = <>, + couch_util:hmac(HashAlgorithm, FullSecret, Message) + end. + +verify(Message, ExpectedMAC) -> + verify(Message, <<>>, ExpectedMAC). + +verify(Message, ExtraSecret, ExpectedMAC) -> + FullSecrets = [<> || Secret <- all_secrets()], + AllAlgorithms = couch_util:get_config_hash_algorithms(), + verify(Message, AllAlgorithms, FullSecrets, ExpectedMAC). + +verify(Message, AllAlgorithms, FullSecrets, ExpectedMAC) -> + Algorithms = lists:filter( + fun(Algorithm) -> + #{size := Size} = crypto:hash_info(Algorithm), + Size == byte_size(ExpectedMAC) + end, + AllAlgorithms + ), + VerifyFun = fun({Secret, Algorithm}) -> + ActualMAC = couch_util:hmac(Algorithm, Secret, Message), + crypto:hash_equals(ExpectedMAC, ActualMAC) + end, + lists:any(VerifyFun, [{S, A} || S <- FullSecrets, A <- Algorithms]). + +secret_is_set() -> + current_secret_from_ets() /= undefined. + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, nil, []). + +init(nil) -> + ets:new(?MODULE, [named_table, {read_concurrency, true}]), + true = ets:insert(?MODULE, {{node(), current}, current_secret_from_config()}), + update_all_secrets(), + erlang:send_after(5000, self(), cache_cleanup), + ok = config:listen_for_changes(?MODULE, undefined), + {ok, nil, {continue, get_secrets}}. + +handle_call({insert, {Node, current}, Secret}, _From, State) -> + case current_secret_from_ets(Node) of + undefined -> + ets:insert(?MODULE, [{{Node, current}, Secret}]); + OldSecret -> + TimeoutSecs = chttpd_util:get_chttpd_auth_config_integer("timeout", 600), + ExpiresAt = erlang:system_time(second) + TimeoutSecs, + ets:insert(?MODULE, [{{Node, current}, Secret}, {{Node, ExpiresAt}, OldSecret}]) + end, + update_all_secrets(), + {reply, ok, State}; +handle_call({insert, Key, Secret}, _From, State) -> + ets:insert(?MODULE, {Key, Secret}), + update_all_secrets(), + {reply, ok, State}; +handle_call(get_secrets, _From, State) -> + Secrets = ets:match_object(?MODULE, {{node(), '_'}, '_'}), + {reply, Secrets, State}; +handle_call(flush_cache, _From, State) -> + %% used from tests to prevent spurious failures due to timing + MatchSpec = [{{{'_', '$1'}, '_'}, [{is_integer, '$1'}], [true]}], + NumDeleted = ets:select_delete(?MODULE, MatchSpec), + if + NumDeleted > 0 -> update_all_secrets(); + true -> ok + end, + {reply, NumDeleted, State}; +handle_call(_Msg, _From, State) -> + {noreply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_continue(get_secrets, State) -> + {Replies, _BadNodes} = gen_server:multi_call(nodes(), ?MODULE, get_secrets), + {_Nodes, Secrets} = lists:unzip(Replies), + true = ets:insert(?MODULE, lists:flatten(Secrets)), + update_all_secrets(), + {noreply, State}. + +handle_info(restart_config_listener, State) -> + ok = config:listen_for_changes(?MODULE, nil), + update_current_secret(), + {noreply, State}; +handle_info(cache_cleanup, State) -> + erlang:send_after(5000, self(), cache_cleanup), + Now = os:system_time(second), + MatchSpec = [{{{'_', '$1'}, '_'}, [{is_integer, '$1'}, {'<', '$1', Now}], [true]}], + NumDeleted = ets:select_delete(?MODULE, MatchSpec), + if + NumDeleted > 0 -> update_all_secrets(); + true -> ok + end, + {noreply, State}; +handle_info(_Msg, State) -> + {noreply, State}. + +handle_config_change("chttpd_auth", "secret", _, _, _) -> + update_current_secret(), + {ok, undefined}; +handle_config_change("couch_httpd_auth", "secret", _, _, _) -> + update_current_secret(), + {ok, undefined}; +handle_config_change(_, _, _, _, _) -> + {ok, undefined}. + +handle_config_terminate(_, stop, _) -> + ok; +handle_config_terminate(_Server, _Reason, _State) -> + erlang:send_after(3000, whereis(?MODULE), restart_config_listener). + +%% private functions + +update_current_secret() -> + NewSecret = current_secret_from_config(), + spawn(fun() -> + gen_server:multi_call(nodes(), ?MODULE, {insert, {node(), current}, NewSecret}), + gen_server:call(?MODULE, {insert, {node(), current}, NewSecret}) + end). + +update_all_secrets() -> + AllSecrets = ets:match_object(?MODULE, {{'_', '_'}, '_'}), + ets:insert(?MODULE, {all_secrets, lists:usort([V || {_K, V} <- AllSecrets, is_binary(V)])}). + +current_secret_from_config() -> + case chttpd_util:get_chttpd_auth_config("secret") of + undefined -> + undefined; + Secret -> + ?l2b(Secret) + end. + +current_secret() -> + case current_secret_from_ets() of + undefined -> + current_secret_from_config(); + CurrentSecret -> + CurrentSecret + end. + +current_secret_from_ets() -> + current_secret_from_ets(node()). + +current_secret_from_ets(Node) -> + secret_from_ets({Node, current}). + +all_secrets() -> + case all_secrets_from_ets() of + [] -> + CurrentSecret = current_secret_from_config(), + if + CurrentSecret == undefined -> []; + true -> [CurrentSecret] + end; + AllSecrets -> + AllSecrets + end. + +all_secrets_from_ets() -> + secret_from_ets(all_secrets). + +secret_from_ets(Key) -> + case ets:lookup(?MODULE, Key) of + [{Key, Value}] -> Value; + [] -> undefined + end. diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index 4b66993f809..d1118bcb5f0 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -302,11 +302,17 @@ init([N]) -> "couchdb", "update_lru_on_read", false ), ok = config:listen_for_changes(?MODULE, N), - % Spawn async .deleted files recursive cleaner, but only - % for the first sharded couch_server instance + case N of - 1 -> ok = couch_file:init_delete_dir(RootDir); - _ -> ok + 1 -> + % Update mm and upgrade_in_progress stats gauges + update_maintenance_mode_gauge(), + update_upgrade_in_progress_gauge(), + % Spawn async .deleted files recursive cleaner, but only + % for the first sharded couch_server instance + ok = couch_file:init_delete_dir(RootDir); + _ -> + ok end, ets:new(couch_dbs(N), [ set, @@ -395,6 +401,12 @@ handle_config_change("httpd", "port", _, _, 1 = N) -> handle_config_change("httpd", "max_connections", _, _, 1 = N) -> couch_httpd:stop(), {ok, N}; +handle_config_change("couchdb", "maintenance_mode", _, _, 1 = N) -> + update_maintenance_mode_gauge(), + {ok, N}; +handle_config_change("couchdb", "upgrade_in_progress", _, _, 1 = N) -> + update_upgrade_in_progress_gauge(), + {ok, N}; handle_config_change(_, _, _, _, N) -> {ok, N}. @@ -1018,6 +1030,19 @@ try_lock(Table, DbName) when is_atom(Table), is_binary(DbName) -> unlock(Table, DbName) when is_atom(Table), is_binary(DbName) -> ets:update_element(Table, DbName, {#entry.lock, unlocked}). +update_maintenance_mode_gauge() -> + % Note, it's not necessarily a boolean, could be "nolb" as well + case config:get("couchdb", "maintenance_mode", "false") of + "false" -> couch_stats:update_gauge([couchdb, maintenance_mode], 0); + _ -> couch_stats:update_gauge([couchdb, maintenance_mode], 1) + end. + +update_upgrade_in_progress_gauge() -> + case config:get_boolean("couchdb", "upgrade_in_progress", false) of + false -> couch_stats:update_gauge([couchdb, upgrade_in_progress], 0); + true -> couch_stats:update_gauge([couchdb, upgrade_in_progress], 1) + end. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/couch/src/couch_stream.erl b/src/couch/src/couch_stream.erl index 0e4ccdae656..f3f0bf19018 100644 --- a/src/couch/src/couch_stream.erl +++ b/src/couch/src/couch_stream.erl @@ -58,7 +58,8 @@ open({_StreamEngine, _StreamEngineState} = Engine) -> open(Engine, []). open({_StreamEngine, _StreamEngineState} = Engine, Options) -> - gen_server:start_link(?MODULE, {Engine, self(), erlang:get(io_priority), Options}, []). + GenOpts = couch_util:hibernate_after(?MODULE), + gen_server:start_link(?MODULE, {Engine, self(), erlang:get(io_priority), Options}, GenOpts). close(Pid) -> gen_server:call(Pid, close, infinity). @@ -223,17 +224,15 @@ handle_call({write, Bin}, _From, Stream) -> Md5_2 = couch_hash:md5_hash_update(Md5, WriteBin2) end, - {reply, ok, - Stream#stream{ - engine = NewEngine, - written_len = WrittenLen2, - buffer_list = [], - buffer_len = 0, - md5 = Md5_2, - identity_md5 = IdenMd5_2, - identity_len = IdenLen + BinSize - }, - hibernate}; + {reply, ok, Stream#stream{ + engine = NewEngine, + written_len = WrittenLen2, + buffer_list = [], + buffer_len = 0, + md5 = Md5_2, + identity_md5 = IdenMd5_2, + identity_len = IdenLen + BinSize + }}; true -> {reply, ok, Stream#stream{ buffer_list = [Bin | Buffer], diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index d93aaebd61b..8ee165666c4 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -46,6 +46,7 @@ -export([remove_sensitive_data/1]). -export([ejson_to_map/1]). -export([new_set/0, set_from_list/1]). +-export([hibernate_after/1]). -include_lib("couch/include/couch_db.hrl"). @@ -64,6 +65,8 @@ <<"feature_flags">> ]). +-define(DEFAULT_HIBERNATE_AFTER, 5000). + priv_dir() -> case code:priv_dir(couch) of {error, bad_name} -> @@ -806,3 +809,13 @@ new_set() -> set_from_list(KVs) -> sets:from_list(KVs, [{version, 2}]). + +hibernate_after(Module) when is_atom(Module) -> + Key = atom_to_list(Module), + Default = ?DEFAULT_HIBERNATE_AFTER, + case config:get_integer_or_infinity("hibernate_after", Key, Default) of + infinity -> + []; + Timeout when is_integer(Timeout) -> + [{hibernate_after, Timeout}] + end. diff --git a/src/couch/src/couch_work_queue.erl b/src/couch/src/couch_work_queue.erl index 3c6ffeaf80a..6ec9d796ca8 100644 --- a/src/couch/src/couch_work_queue.erl +++ b/src/couch/src/couch_work_queue.erl @@ -34,7 +34,8 @@ }). new(Options) -> - gen_server:start_link(couch_work_queue, Options, []). + GenOpts = couch_util:hibernate_after(?MODULE), + gen_server:start_link(couch_work_queue, Options, GenOpts). queue(Wq, Item) when is_binary(Item) -> gen_server:call(Wq, {queue, Item, byte_size(Item)}, infinity); @@ -73,7 +74,7 @@ init(Options) -> max_size = couch_util:get_value(max_size, Options, nil), max_items = couch_util:get_value(max_items, Options, nil) }, - {ok, Q, hibernate}. + {ok, Q}. terminate(_Reason, #q{worker = undefined}) -> ok; @@ -87,18 +88,13 @@ handle_call({queue, Item, Size}, From, #q{worker = undefined} = Q0) -> items = Q0#q.items + 1, queue = queue:in({Item, Size}, Q0#q.queue) }, - case - (Q#q.size >= Q#q.max_size) orelse - (Q#q.items >= Q#q.max_items) - of - true -> - {noreply, Q#q{blocked = [From | Q#q.blocked]}, hibernate}; - false -> - {reply, ok, Q, hibernate} + case (Q#q.size >= Q#q.max_size) orelse (Q#q.items >= Q#q.max_items) of + true -> {noreply, Q#q{blocked = [From | Q#q.blocked]}}; + false -> {reply, ok, Q} end; handle_call({queue, Item, _}, _From, #q{worker = {W, _Max}} = Q) -> gen_server:reply(W, {ok, [Item]}), - {reply, ok, Q#q{worker = undefined}, hibernate}; + {reply, ok, Q#q{worker = undefined}}; handle_call({dequeue, _Max}, _From, #q{worker = {_, _}}) -> % Something went wrong - the same or a different worker is % trying to dequeue an item. We only allow one worker to wait diff --git a/src/couch/src/test_util.erl b/src/couch/src/test_util.erl index b8001840fb9..fd5364fb74a 100644 --- a/src/couch/src/test_util.erl +++ b/src/couch/src/test_util.erl @@ -125,6 +125,7 @@ start_applications([App | Apps], Acc) -> stop_applications(Apps) -> [application:stop(App) || App <- lists:reverse(Apps)], + ok = application:unset_env(config, ini_files), ok. start_config(Chain) -> @@ -374,7 +375,7 @@ mock(couch_stats) -> ok. load_applications_with_stats() -> - Wildcard = filename:join([?BUILDDIR(), "src/*/priv/stats_descriptions.cfg"]), + Wildcard = ?ABS_PATH("../*/priv/stats_descriptions.cfg"), [application:load(stats_file_to_app(File)) || File <- filelib:wildcard(Wildcard)], ok. diff --git a/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl b/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl index a42ef52c506..38b325022f7 100644 --- a/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl +++ b/src/couch/test/eunit/couch_auto_purge_plugin_tests.erl @@ -29,7 +29,9 @@ couch_quickjs_scanner_plugin_test_() -> ?TDEF_FE(t_min_batch_size_1, 10), ?TDEF_FE(t_min_batch_size_2, 10), ?TDEF_FE(t_max_batch_size_1, 10), - ?TDEF_FE(t_max_batch_size_2, 10) + ?TDEF_FE(t_max_batch_size_2, 10), + ?TDEF_FE(t_dry_run, 10), + ?TDEF_FE(t_dry_run_with_non_default_log_level, 10) ] }. @@ -42,6 +44,7 @@ setup() -> DbName = ?tempdb(), ok = fabric:create_db(DbName, [{q, "2"}, {n, "1"}]), config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false), + config:set(atom_to_list(?PLUGIN), "dry_run", "false", false), reset_stats(), {Ctx, DbName}. @@ -87,6 +90,37 @@ t_auto_purge_after_db_ttl({_, DbName}) -> ?assertEqual(0, doc_del_count(DbName)), ok. +t_dry_run({_, DbName}) -> + config:set(atom_to_list(?PLUGIN), "dry_run", "true", false), + config:set(atom_to_list(?PLUGIN), "deleted_document_ttl", "-3_hour", false), + ok = add_doc(DbName, <<"doc1">>, #{<<"_deleted">> => true}), + ?assertEqual(1, doc_del_count(DbName)), + meck:reset(couch_scanner_server), + meck:reset(couch_scanner_util), + meck:reset(?PLUGIN), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, doc_del_count(DbName)), + ?assert(log_calls(info) >= 3), + ?assert(log_calls(warning) < 3), + ok. + +t_dry_run_with_non_default_log_level({_, DbName}) -> + config:set(atom_to_list(?PLUGIN), "dry_run", "true", false), + config:set(atom_to_list(?PLUGIN), "log_level", "warning", false), + config:set(atom_to_list(?PLUGIN), "deleted_document_ttl", "-3_hour", false), + ok = add_doc(DbName, <<"doc1">>, #{<<"_deleted">> => true}), + ?assertEqual(1, doc_del_count(DbName)), + meck:reset(couch_scanner_server), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, doc_del_count(DbName)), + ?assert(log_calls(warning) >= 3), + ?assert(log_calls(info) < 3), + ok. + t_min_batch_size_1({_, DbName}) -> meck:new(fabric, [passthrough]), config:set_integer(atom_to_list(?PLUGIN), "min_batch_size", 5), @@ -210,3 +244,6 @@ wait_exit(MSec) -> doc_del_count(DbName) -> {ok, DbInfo} = fabric:get_db_info(DbName), couch_util:get_value(doc_del_count, DbInfo). + +log_calls(Level) -> + meck:num_calls(couch_scanner_util, log, [Level, ?PLUGIN, '_', '_', '_']). diff --git a/src/couch/test/eunit/couch_passwords_hasher_tests.erl b/src/couch/test/eunit/couch_passwords_hasher_tests.erl new file mode 100644 index 00000000000..9c892537632 --- /dev/null +++ b/src/couch/test/eunit/couch_passwords_hasher_tests.erl @@ -0,0 +1,207 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_passwords_hasher_tests). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). + +-define(USER, "couch_passwords_hash_test_admin"). +-define(PASS, "pass"). +-define(AUTH, {basic_auth, {?USER, ?PASS}}). +-define(CONTENT_JSON, {"Content-Type", "application/json"}). +-define(RANDOM_USER, "user-" ++ ?b2l(couch_uuids:random())). +-define(MOD, couch_password_hasher). + +setup(Scheme) -> + Hashed = couch_passwords:hash_admin_password(?PASS), + config:set("admins", ?USER, ?b2l(Hashed), false), + Db = ?b2l(?tempdb()), + create_db(Db), + config:set("chttpd_auth", "authentication_db", Db, false), + config:set("chttpd_auth", "password_scheme", Scheme, false), + meck:new(?MOD, [passthrough]), + Db. + +teardown(_, Db) -> + delete_db(Db), + config:delete("admins", ?USER, false), + config:delete("chttpd_auth", "authentication_db", false), + config:delete("chttpd_auth", "password_scheme", false), + meck:unload(). + +couch_password_hasher_test_() -> + { + "couch_password_hasher tests", + { + setup, + fun() -> test_util:start_couch([chttpd]) end, + fun test_util:stop_couch/1, + [ + upgrade_password_hash_tests("simple"), + upgrade_password_hash_tests("pbkdf2") + ] + } + }. + +upgrade_password_hash_tests(Scheme) -> + { + "password scheme " ++ Scheme ++ " tests", + foreachx, + fun setup/1, + fun teardown/2, + [ + {Scheme, Test} + || Test <- + [ + fun create_user_by_admin_should_not_upgrade_password_hash/2, + fun request_by_user_should_not_upgrade_password_hash/2, + fun update_user_password_by_user_should_not_upgrade_password_hash/2 + ] + ] + }. + +create_user_by_admin_should_not_upgrade_password_hash(_, Db) -> + ?_test(begin + meck:reset(?MOD), + User = ?RANDOM_USER, + create_user(Db, User, ?PASS), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ) + end). + +request_by_user_should_not_upgrade_password_hash(_, Db) -> + ?_test(begin + User = ?RANDOM_USER, + create_user(Db, User, ?PASS), + {200, _} = req(get, url(Db, "org.couchdb.user:" ++ User)), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ), + + meck:reset(?MOD), + Headers = [{basic_auth, {User, ?PASS}}], + {200, _} = req(get, url(), Headers, []), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ) + end). + +update_user_password_by_user_should_not_upgrade_password_hash(_, Db) -> + ?_test(begin + User = ?RANDOM_USER, + ?debugVal(User), + create_user(Db, User, ?PASS), + {200, #{<<"_rev">> := Rev}} = req(get, url(Db, "org.couchdb.user:" ++ User)), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ), + + meck:reset(?MOD), + NewPass = "new_password", + update_password(Db, User, NewPass, ?b2l(Rev)), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ), + + OldAuth = [{basic_auth, {User, ?PASS}}], + {401, _} = req(get, url(), OldAuth, []), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ), + NewAuth = [{basic_auth, {User, NewPass}}], + {200, _} = req(get, url(), NewAuth, []), + ?assertNot( + meck:called(?MOD, handle_cast, [ + {upgrade_password_hash, '_', ?l2b(User), '_', '_', '_', '_'}, '_' + ]) + ) + end). + +%%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%% +url() -> + Addr = config:get("chttpd", "bind_address", "127.0.0.1"), + Port = mochiweb_socket_server:get(chttpd, port), + lists:concat(["http://", Addr, ":", Port]). + +url(Db) -> + url() ++ "/" ++ Db. + +url(Db, Path) -> + url(Db) ++ "/" ++ Path. + +create_db(Db) -> + case req(put, url(Db)) of + {201, #{}} -> ok; + Error -> error({failed_to_create_test_db, Db, Error}) + end. + +delete_db(Db) -> + case req(delete, url(Db)) of + {200, #{}} -> ok; + Error -> error({failed_to_delete_test_db, Db, Error}) + end. + +create_user(Db, UserName, Password) -> + ok = couch_auth_cache:ensure_users_db_exists(), + User = ?l2b(UserName), + Pass = ?l2b(Password), + case req(put, url(Db, "org.couchdb.user:" ++ UserName), user_doc(User, Pass)) of + {201, #{}} -> ok; + Error -> error({failed_to_create_user, UserName, Error}) + end. + +update_password(Db, UserName, NewPassword, Rev) -> + User = ?l2b(UserName), + NewPass = ?l2b(NewPassword), + Headers = [?AUTH, {"If-Match", Rev}], + case req(put, url(Db, "org.couchdb.user:" ++ UserName), Headers, user_doc(User, NewPass)) of + {201, #{}} -> ok; + Error -> error({failed_to_update_password, UserName, Error}) + end. + +user_doc(User, Pass) -> + jiffy:encode( + {[ + {<<"name">>, User}, + {<<"password">>, Pass}, + {<<"roles">>, []}, + {<<"type">>, <<"user">>} + ]} + ). + +req(Method, Url) -> + Headers = [?CONTENT_JSON, ?AUTH], + {ok, Code, _, Res} = test_request:request(Method, Url, Headers), + {Code, jiffy:decode(Res, [return_maps])}. + +req(Method, Url, Body) -> + Headers = [?CONTENT_JSON, ?AUTH], + {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body), + {Code, jiffy:decode(Res, [return_maps])}. + +req(Method, Url, Headers, Body) -> + {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body), + {Code, jiffy:decode(Res, [return_maps])}. diff --git a/src/couch/test/eunit/couch_secrets_tests.erl b/src/couch/test/eunit/couch_secrets_tests.erl new file mode 100644 index 00000000000..37305a63bfa --- /dev/null +++ b/src/couch/test/eunit/couch_secrets_tests.erl @@ -0,0 +1,81 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_secrets_tests). + +-include_lib("couch/include/couch_eunit.hrl"). + +-define(DATA1, <<"data1">>). +-define(DATA2, <<"data2">>). +-define(BADMAC, <<"badmac">>). +-define(EXTRA1, <<"extra1">>). +-define(EXTRA2, <<"extra2">>). +-define(SECRET1, "secret1"). +-define(SECRET2, "secret2"). + +couch_secrets_test_() -> + {setup, fun test_util:start_couch/0, fun test_util:stop_couch/1, + {with, [ + fun error_if_no_secret/1, + fun verify_works/1, + fun verify_extra_secret_works/1, + fun verify_old_secret_works/1, + fun verify_old_secret_stops_working/1 + ]}}. + +error_if_no_secret(_Ctx) -> + delete_secret(), + ?assertThrow( + {internal_server_error, <<"cookie auth secret is not set">>}, couch_secrets:sign(?DATA1) + ). + +verify_works(_Ctx) -> + set_secret(?SECRET1), + MAC = couch_secrets:sign(?DATA1), + ?assert(couch_secrets:verify(?DATA1, MAC)), + ?assertNot(couch_secrets:verify(?DATA1, ?BADMAC)), + ?assertNot(couch_secrets:verify(?DATA2, MAC)). + +verify_extra_secret_works(_Ctx) -> + set_secret(?SECRET1), + MAC = couch_secrets:sign(?DATA1, ?EXTRA1), + ?assert(couch_secrets:verify(?DATA1, ?EXTRA1, MAC)), + ?assertNot(couch_secrets:verify(?DATA1, ?EXTRA2, MAC)), + ?assertNot(couch_secrets:verify(?DATA1, ?EXTRA1, ?BADMAC)), + ?assertNot(couch_secrets:verify(?DATA2, ?EXTRA1, MAC)). + +verify_old_secret_works(_Ctx) -> + set_secret(?SECRET1), + MAC1 = couch_secrets:sign(?DATA1), + set_secret(?SECRET2), + MAC2 = couch_secrets:sign(?DATA1), + ?assert(couch_secrets:verify(?DATA1, MAC1)), + ?assert(couch_secrets:verify(?DATA1, MAC2)). + +verify_old_secret_stops_working(_Ctx) -> + set_secret(?SECRET1), + MAC1 = couch_secrets:sign(?DATA1), + ?assert(couch_secrets:verify(?DATA1, MAC1)), + set_secret(?SECRET2), + MAC2 = couch_secrets:sign(?DATA1), + ?assert(couch_secrets:verify(?DATA1, MAC2)), + ?assert(gen_server:call(couch_secrets, flush_cache) > 0), + ?assertNot(couch_secrets:verify(?DATA1, MAC1)), + ?assert(couch_secrets:verify(?DATA1, MAC2)). + +delete_secret() -> + config:delete("chttpd_auth", "secret"), + config:delete("couch_httpd_auth", "secret"). + +set_secret(Secret) -> + config:set("chttpd_auth", "secret", Secret), + timer:sleep(100). diff --git a/src/couch/test/eunit/couch_server_tests.erl b/src/couch/test/eunit/couch_server_tests.erl index 77ddfde2131..35e1d539d09 100644 --- a/src/couch/test/eunit/couch_server_tests.erl +++ b/src/couch/test/eunit/couch_server_tests.erl @@ -288,3 +288,43 @@ get_next_message() -> after 5000 -> error(timeout) end. + +get_stats_and_gagues_test_() -> + { + foreach, + fun() -> + Ctx = test_util:start_couch(), + config:set("couchdb", "maintenance_mode", "false", false), + config:set("couchdb", "upgrade_in_progress", "false", false), + Ctx + end, + fun(Ctx) -> + config:delete("couchdb", "maintenance_mode", false), + config:delete("couchdb", "upgrade_in_progress", false), + test_util:stop(Ctx) + end, + [ + ?TDEF_FE(t_maintenance_mode_metric), + ?TDEF_FE(t_upgrade_in_progress_metric), + ?TDEF_FE(t_get_stats) + ] + }. + +t_maintenance_mode_metric(_) -> + ?assertEqual(0, couch_stats:sample([couchdb, maintenance_mode])), + config:set("couchdb", "maintenance_mode", "true", false), + ?assertEqual(1, couch_stats:sample([couchdb, maintenance_mode])), + config:set("couchdb", "maintenance_mode", "nolb", false), + ?assertEqual(1, couch_stats:sample([couchdb, maintenance_mode])), + config:set("couchdb", "maintenance_mode", "false", false), + ?assertEqual(0, couch_stats:sample([couchdb, maintenance_mode])). + +t_upgrade_in_progress_metric(_) -> + ?assertEqual(0, couch_stats:sample([couchdb, upgrade_in_progress])), + config:set("couchdb", "upgrade_in_progress", "true", false), + ?assertEqual(1, couch_stats:sample([couchdb, upgrade_in_progress])), + config:set("couchdb", "upgrade_in_progress", "false", false), + ?assertEqual(0, couch_stats:sample([couchdb, upgrade_in_progress])). + +t_get_stats(_) -> + ?assertMatch([{start_time, _}, {dbs_open, _}], couch_server:get_stats()). diff --git a/src/couch/test/eunit/couch_util_tests.erl b/src/couch/test/eunit/couch_util_tests.erl index 5f8f1ce4323..ff40a6fddfc 100644 --- a/src/couch/test/eunit/couch_util_tests.erl +++ b/src/couch/test/eunit/couch_util_tests.erl @@ -206,3 +206,83 @@ ejson_to_map_test() -> ?assertEqual(#{a => 1, b => 2}, couch_util:ejson_to_map({[{b, 2}, {a, 1}]})), ?assertEqual(#{<<"a">> => [1, #{}]}, couch_util:ejson_to_map({[{<<"a">>, [1, {[]}]}]})), ?assertEqual([#{true => 1}], couch_util:ejson_to_map([{[{true, 1}]}])). + +hibernate_after_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_hibernate), + ?TDEF_FE(t_do_not_hibernate) + ] + }. + +setup() -> + test_util:start_applications([config]). + +teardown(Ctx) -> + config:delete("hibernate_after", "couch_work_queue", false), + test_util:stop_applications(Ctx). + +t_hibernate(_) -> + ?assertEqual([{hibernate_after, 5000}], hibernate_cfg()), + + % Set non-default value + config:set("hibernate_after", "couch_work_queue", "100", false), + ?assertEqual([{hibernate_after, 100}], hibernate_cfg()), + + {ok, Q} = couch_work_queue:new([]), + % Enqueue item, creating gen_server activity + ok = couch_work_queue:queue(Q, potato), + + % Assert that we eventually hibernate + wait_hibernate(Q), + {current_function, {_, F, _}} = process_info(Q, current_function), + % Note: different versions of OTP hibernate at a different function + ?assert(hibernate == F orelse loop_hibernate == F), + + % We can wait wake up from hibernation and get back to work + ?assertEqual({ok, [potato]}, couch_work_queue:dequeue(Q)), + + % Then we can get back into hibernatation + wait_hibernate(Q), + {current_function, {_, F, _}} = process_info(Q, current_function), + ?assert(hibernate == F orelse loop_hibernate == F), + + couch_work_queue:close(Q). + +t_do_not_hibernate(_) -> + config:set("hibernate_after", "couch_work_queue", "infinity", false), + ?assertEqual([], hibernate_cfg()), + + {ok, Q} = couch_work_queue:new([]), + + % Enqueue item, creating some gen_server activity + ok = couch_work_queue:queue(Q, potato), + + % Assert that we do not hibernate + timer:sleep(200), + {current_function, {_, F, _}} = process_info(Q, current_function), + ?assertNot(hibernate == F orelse loop_hibernate == F), + + % The queue works as expected without hibernation + ?assertEqual({ok, [potato]}, couch_work_queue:dequeue(Q)), + + couch_work_queue:close(Q). + +% Helper functions + +hibernate_cfg() -> + couch_util:hibernate_after(couch_work_queue). + +wait_hibernate(Pid) -> + WaitFun = fun() -> + {current_function, {M, F, _}} = process_info(Pid, current_function), + case {M, F} of + {erlang, hibernate} -> ok; + {gen_server, loop_hibernate} -> ok; + {_, _} -> wait + end + end, + test_util:wait(WaitFun). diff --git a/src/couch/test/eunit/fixtures/os_daemon_bad_perm.sh b/src/couch/test/eunit/fixtures/os_daemon_bad_perm.sh deleted file mode 100644 index 345c8b40b26..00000000000 --- a/src/couch/test/eunit/fixtures/os_daemon_bad_perm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -e -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -# Please do not make this file executable as that's the error being tested. - -sleep 5 diff --git a/src/couch/test/eunit/fixtures/os_daemon_can_reboot.sh b/src/couch/test/eunit/fixtures/os_daemon_can_reboot.sh deleted file mode 100755 index 5bc10e83f16..00000000000 --- a/src/couch/test/eunit/fixtures/os_daemon_can_reboot.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -e -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -sleep 2 diff --git a/src/couch/test/eunit/fixtures/os_daemon_configer.escript b/src/couch/test/eunit/fixtures/os_daemon_configer.escript deleted file mode 100755 index f146b831420..00000000000 --- a/src/couch/test/eunit/fixtures/os_daemon_configer.escript +++ /dev/null @@ -1,97 +0,0 @@ -#! /usr/bin/env escript - -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --include("../../include/couch_eunit.hrl"). - -read() -> - case io:get_line('') of - eof -> - stop; - Data -> - jiffy:decode(Data) - end. - -write(Mesg) -> - Data = iolist_to_binary(jiffy:encode(Mesg)), - io:format(binary_to_list(Data) ++ "\n", []). - -get_cfg(Section) -> - write([<<"get">>, Section]), - read(). - -get_cfg(Section, Name) -> - write([<<"get">>, Section, Name]), - read(). - -log(Mesg) -> - write([<<"log">>, Mesg]). - -log(Mesg, Level) -> - write([<<"log">>, Mesg, {[{<<"level">>, Level}]}]). - -test_get_cfg1() -> - Path = list_to_binary(?FILE), - FileName = list_to_binary(filename:basename(?FILE)), - {[{FileName, Path}]} = get_cfg(<<"os_daemons">>). - -test_get_cfg2() -> - Path = list_to_binary(?FILE), - FileName = list_to_binary(filename:basename(?FILE)), - Path = get_cfg(<<"os_daemons">>, FileName), - <<"sequential">> = get_cfg(<<"uuids">>, <<"algorithm">>). - - -test_get_unknown_cfg() -> - {[]} = get_cfg(<<"aal;3p4">>), - null = get_cfg(<<"aal;3p4">>, <<"313234kjhsdfl">>). - -test_log() -> - log(<<"foobar!">>), - log(<<"some stuff!">>, <<"debug">>), - log(2), - log(true), - write([<<"log">>, <<"stuff">>, 2]), - write([<<"log">>, 3, null]), - write([<<"log">>, [1, 2], {[{<<"level">>, <<"debug">>}]}]), - write([<<"log">>, <<"true">>, {[]}]). - -do_tests() -> - test_get_cfg1(), - test_get_cfg2(), - test_get_unknown_cfg(), - test_log(), - loop(io:read("")). - -loop({ok, _}) -> - loop(io:read("")); -loop(eof) -> - init:stop(); -loop({error, _Reason}) -> - init:stop(). - -main([]) -> - init_code_path(), - do_tests(). - -init_code_path() -> - Paths = [ - "couchdb", - "jiffy", - "ibrowse", - "mochiweb", - "snappy" - ], - lists:foreach(fun(Name) -> - code:add_patha(filename:join([?BUILDDIR(), "src", Name, "ebin"])) - end, Paths). diff --git a/src/couch/test/eunit/fixtures/os_daemon_die_quickly.sh b/src/couch/test/eunit/fixtures/os_daemon_die_quickly.sh deleted file mode 100755 index f5a13684ecd..00000000000 --- a/src/couch/test/eunit/fixtures/os_daemon_die_quickly.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -e -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -sleep 1 diff --git a/src/couch/test/eunit/fixtures/os_daemon_looper.escript b/src/couch/test/eunit/fixtures/os_daemon_looper.escript deleted file mode 100755 index 73974e905a7..00000000000 --- a/src/couch/test/eunit/fixtures/os_daemon_looper.escript +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env escript - -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - -loop() -> - loop(io:read("")). - -loop({ok, _}) -> - loop(io:read("")); -loop(eof) -> - stop; -loop({error, Reason}) -> - throw({error, Reason}). - -main([]) -> - loop(). diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl index 6ba226aad92..ae1e0288f4e 100644 --- a/src/couch_mrview/include/couch_mrview.hrl +++ b/src/couch_mrview/include/couch_mrview.hrl @@ -30,7 +30,8 @@ doc_queue, write_queue, qserver=nil, - view_info=#{} + view_info=#{}, + start_time }). diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl index a534dd0e8a9..601035901a9 100644 --- a/src/couch_mrview/src/couch_mrview_compactor.erl +++ b/src/couch_mrview/src/couch_mrview_compactor.erl @@ -129,7 +129,8 @@ compact(State) -> {ok, EmptyState#mrst{ id_btree = NewIdBtree, views = NewViews, - update_seq = Seq + update_seq = Seq, + start_time = now_secs() }}. recompact(State) -> @@ -248,10 +249,12 @@ swap_compacted(OldState, NewState) -> {ok, Pre} = couch_file:bytes(Fd), {ok, Post} = couch_file:bytes(NewFd), - couch_log:notice("Compaction swap for view ~s ~p ~p", [ + Duration = now_secs() - NewState#mrst.start_time, + couch_log:notice("Compaction swap for view ~s ~p ~p ~Bs", [ IndexFName, Pre, - Post + Post, + Duration ]), ok = couch_file:delete(RootDir, IndexFName), ok = file:rename(CompactFName, IndexFName), @@ -259,7 +262,7 @@ swap_compacted(OldState, NewState) -> unlink(OldState#mrst.fd), demonitor(OldState#mrst.fd_monitor, [flush]), - {ok, NewState#mrst{fd_monitor = Ref}}. + {ok, NewState#mrst{fd_monitor = Ref, start_time = now_secs()}}. remove_compacted(#mrst{sig = Sig, db_name = DbName} = State) -> RootDir = couch_index_util:root_dir(), @@ -267,6 +270,9 @@ remove_compacted(#mrst{sig = Sig, db_name = DbName} = State) -> ok = couch_file:delete(RootDir, CompactFName), {ok, State}. +now_secs() -> + erlang:monotonic_time(second). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 5405e8db826..494f5b3ec68 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -61,6 +61,8 @@ (C >= $A andalso C =< $F)) ). +-define(DEFAULT_BTREE_CACHE_DEPTH, 3). + -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). @@ -369,7 +371,8 @@ init_state(Db, Fd, State, Header) -> }} = maybe_update_header(Header), IdBtOpts = [ - {compression, couch_compress:get_compression_method()} + {compression, couch_compress:get_compression_method()}, + {cache_depth, btree_cache_depth()} ], {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts), @@ -394,7 +397,8 @@ open_view(_Db, Fd, Lang, ViewState, View) -> ViewBtOpts = [ {less, LessFun}, {reduce, ReduceFun}, - {compression, Compression} + {compression, Compression}, + {cache_depth, btree_cache_depth()} ], {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts), @@ -1365,3 +1369,6 @@ compact_on_collator_upgrade() -> commit_on_header_upgrade() -> config:get_boolean("view_upgrade", "commit_on_header_upgrade", true). + +btree_cache_depth() -> + config:get_integer("bt_engine_cache", "view_btree_cache_depth", ?DEFAULT_BTREE_CACHE_DEPTH). diff --git a/src/couch_quickjs/build_js.escript b/src/couch_quickjs/build_js.escript index 1da48ee876e..49aec0f1a25 100644 --- a/src/couch_quickjs/build_js.escript +++ b/src/couch_quickjs/build_js.escript @@ -74,7 +74,7 @@ compile_bytecode(Js, CBytecode) -> Tmp = CBytecode ++ ".tmp", {ok, Cwd} = file:get_cwd(), CompileCmd = Cwd ++ "/quickjs/qjsc -c -N bytecode -o c_src/" ++ Tmp ++ " priv/" ++ Js, - os:cmd(CompileCmd), + os:cmd(CompileCmd, #{exception_on_failure => true}), Changed = cp_if_different("c_src/" ++ Tmp, "c_src/" ++ CBytecode), rm("c_src/" ++ Tmp), Changed. diff --git a/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch b/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch index 2f642114eab..db740bf2bf6 100644 --- a/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch +++ b/src/couch_quickjs/patches/01-spidermonkey-185-mode.patch @@ -1,6 +1,6 @@ ---- quickjs-master/quickjs.c 2025-11-29 09:14:41.000000000 -0500 -+++ quickjs/quickjs.c 2025-11-29 20:38:38.829402534 -0500 -@@ -31420,10 +31420,24 @@ +--- quickjs-master/quickjs.c 2025-12-22 09:12:46 ++++ quickjs/quickjs.c 2026-01-04 23:55:00 +@@ -31443,10 +31443,24 @@ if (s->token.val == TOK_FUNCTION || (token_is_pseudo_keyword(s, JS_ATOM_async) && peek_token(s, TRUE) == TOK_FUNCTION)) { diff --git a/src/couch_quickjs/patches/02-test262-errors.patch b/src/couch_quickjs/patches/02-test262-errors.patch index 692e080acef..81646c9003f 100644 --- a/src/couch_quickjs/patches/02-test262-errors.patch +++ b/src/couch_quickjs/patches/02-test262-errors.patch @@ -1,5 +1,5 @@ ---- quickjs-master/test262_errors.txt 2025-11-29 09:14:41.000000000 -0500 -+++ quickjs/test262_errors.txt 2025-11-29 20:38:38.835402578 -0500 +--- quickjs-master/test262_errors.txt 2025-12-22 09:12:46 ++++ quickjs/test262_errors.txt 2026-01-04 23:55:00 @@ -23,6 +23,8 @@ test262/test/language/module-code/ambiguous-export-bindings/namespace-unambiguous-if-export-star-as-from-and-import-star-as-and-export.js:74: SyntaxError: export 'foo' in module 'test262/test/language/module-code/ambiguous-export-bindings/namespace-unambiguous-if-import-star-as-and-export.js' is ambiguous test262/test/language/module-code/ambiguous-export-bindings/namespace-unambiguous-if-export-star-as-from.js:75: SyntaxError: export 'foo' in module 'test262/test/language/module-code/ambiguous-export-bindings/namespace-unambiguous-if-export-star-as-from.js' is ambiguous diff --git a/src/couch_quickjs/quickjs/Changelog b/src/couch_quickjs/quickjs/Changelog index 070b0a77a81..3c08f0c589c 100644 --- a/src/couch_quickjs/quickjs/Changelog +++ b/src/couch_quickjs/quickjs/Changelog @@ -6,6 +6,7 @@ - added Atomics.pause - added added Map and WeakMap upsert methods - added Math.sumPrecise() +- added regexp duplicate named groups - misc bug fixes 2025-09-13: diff --git a/src/couch_quickjs/quickjs/libregexp-opcode.h b/src/couch_quickjs/quickjs/libregexp-opcode.h index 6b97b127303..b3d7b6fdf3a 100644 --- a/src/couch_quickjs/quickjs/libregexp-opcode.h +++ b/src/couch_quickjs/quickjs/libregexp-opcode.h @@ -31,6 +31,8 @@ DEF(char32, 5) DEF(char32_i, 5) DEF(dot, 1) DEF(any, 1) /* same as dot but match any character including line terminator */ +DEF(space, 1) +DEF(not_space, 1) /* must come after */ DEF(line_start, 1) DEF(line_start_m, 1) DEF(line_end, 1) @@ -54,7 +56,7 @@ DEF(word_boundary, 1) DEF(word_boundary_i, 1) DEF(not_word_boundary, 1) DEF(not_word_boundary_i, 1) -DEF(back_reference, 2) +DEF(back_reference, 2) /* variable length */ DEF(back_reference_i, 2) /* must come after */ DEF(backward_back_reference, 2) /* must come after */ DEF(backward_back_reference_i, 2) /* must come after */ diff --git a/src/couch_quickjs/quickjs/libregexp.c b/src/couch_quickjs/quickjs/libregexp.c index 0c989b96948..c387f004387 100644 --- a/src/couch_quickjs/quickjs/libregexp.c +++ b/src/couch_quickjs/quickjs/libregexp.c @@ -34,7 +34,9 @@ /* TODO: - + - remove REOP_char_i and REOP_range_i by precomputing the case folding. + - add specific opcodes for simple unicode property tests so that the + generated bytecode is smaller. - Add a lock step execution mode (=linear time execution guaranteed) when the regular expression is "simple" i.e. no backreference nor complicated lookahead. The opcodes are designed for this execution @@ -77,6 +79,7 @@ typedef struct { BOOL ignore_case; BOOL multi_line; BOOL dotall; + uint8_t group_name_scope; int capture_count; int total_capture_count; /* -1 = not computed yet */ int has_named_captures; /* -1 = don't know, 0 = no, 1 = yes */ @@ -478,7 +481,7 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, if (i != 1) printf(","); printf("<%s>", p); - p += strlen(p) + 1; + p += strlen(p) + LRE_GROUP_NAME_TRAILER_LEN; } printf("\n"); assert(p == (char *)(buf + buf_len)); @@ -547,11 +550,22 @@ static __maybe_unused void lre_dump_bytecode(const uint8_t *buf, break; case REOP_save_start: case REOP_save_end: + printf(" %u", buf[pos + 1]); + break; case REOP_back_reference: case REOP_back_reference_i: case REOP_backward_back_reference: case REOP_backward_back_reference_i: - printf(" %u", buf[pos + 1]); + { + int n, i; + n = buf[pos + 1]; + len += n; + for(i = 0; i < n; i++) { + if (i != 0) + printf(","); + printf(" %u", buf[pos + 2 + i]); + } + } break; case REOP_save_reset: printf(" %u %u", buf[pos + 1], buf[pos + 2]); @@ -745,9 +759,21 @@ int lre_parse_escape(const uint8_t **pp, int allow_utf16) c = '\v'; break; case 'x': + { + int h0, h1; + + h0 = from_hex(*p++); + if (h0 < 0) + return -1; + h1 = from_hex(*p++); + if (h1 < 0) + return -1; + c = (h0 << 4) | h1; + } + break; case 'u': { - int h, n, i; + int h, i; uint32_t c1; if (*p == '{' && allow_utf16) { @@ -765,14 +791,8 @@ int lre_parse_escape(const uint8_t **pp, int allow_utf16) } p++; } else { - if (c == 'x') { - n = 2; - } else { - n = 4; - } - c = 0; - for(i = 0; i < n; i++) { + for(i = 0; i < 4; i++) { h = from_hex(*p++); if (h < 0) { return -1; @@ -1060,7 +1080,7 @@ static int get_class_atom(REParseState *s, REStringList *cr, goto default_escape; if (cr_init_char_range(s, cr, c)) return -1; - c = CLASS_RANGE_BASE; + c += CLASS_RANGE_BASE; break; case 'c': c = *p; @@ -1531,17 +1551,18 @@ static int re_parse_char_class(REParseState *s, const uint8_t **pp) return -1; } -/* Return: - - true if the opcodes may not advance the char pointer - - false if the opcodes always advance the char pointer +/* need_check_adv: false if the opcodes always advance the char pointer + need_capture_init: true if all the captures in the atom are not set */ -static BOOL re_need_check_advance(const uint8_t *bc_buf, int bc_buf_len) +static BOOL re_need_check_adv_and_capture_init(BOOL *pneed_capture_init, + const uint8_t *bc_buf, int bc_buf_len) { int pos, opcode, len; uint32_t val; - BOOL ret; + BOOL need_check_adv, need_capture_init; - ret = TRUE; + need_check_adv = TRUE; + need_capture_init = FALSE; pos = 0; while (pos < bc_buf_len) { opcode = bc_buf[pos]; @@ -1551,20 +1572,23 @@ static BOOL re_need_check_advance(const uint8_t *bc_buf, int bc_buf_len) case REOP_range_i: val = get_u16(bc_buf + pos + 1); len += val * 4; - goto simple_char; + need_check_adv = FALSE; + break; case REOP_range32: case REOP_range32_i: val = get_u16(bc_buf + pos + 1); len += val * 8; - goto simple_char; + need_check_adv = FALSE; + break; case REOP_char: case REOP_char_i: case REOP_char32: case REOP_char32_i: case REOP_dot: case REOP_any: - simple_char: - ret = FALSE; + case REOP_space: + case REOP_not_space: + need_check_adv = FALSE; break; case REOP_line_start: case REOP_line_start_m: @@ -1582,18 +1606,25 @@ static BOOL re_need_check_advance(const uint8_t *bc_buf, int bc_buf_len) case REOP_save_start: case REOP_save_end: case REOP_save_reset: + break; case REOP_back_reference: case REOP_back_reference_i: case REOP_backward_back_reference: case REOP_backward_back_reference_i: + val = bc_buf[pos + 1]; + len += val; + need_capture_init = TRUE; break; default: /* safe behavior: we cannot predict the outcome */ - return TRUE; + need_capture_init = TRUE; + goto done; } pos += len; } - return ret; + done: + *pneed_capture_init = need_capture_init; + return need_check_adv; } /* '*pp' is the first char after '<' */ @@ -1652,16 +1683,16 @@ static int re_parse_group_name(char *buf, int buf_size, const uint8_t **pp) } /* if capture_name = NULL: return the number of captures + 1. - Otherwise, return the capture index corresponding to capture_name - or -1 if none */ + Otherwise, return the number of matching capture groups */ static int re_parse_captures(REParseState *s, int *phas_named_captures, - const char *capture_name) + const char *capture_name, BOOL emit_group_index) { const uint8_t *p; - int capture_index; + int capture_index, n; char name[TMP_BUF_SIZE]; capture_index = 1; + n = 0; *phas_named_captures = 0; for (p = s->buf_start; p < s->buf_end; p++) { switch (*p) { @@ -1673,8 +1704,11 @@ static int re_parse_captures(REParseState *s, int *phas_named_captures, if (capture_name) { p += 3; if (re_parse_group_name(name, sizeof(name), &p) == 0) { - if (!strcmp(name, capture_name)) - return capture_index; + if (!strcmp(name, capture_name)) { + if (emit_group_index) + dbuf_putc(&s->byte_code, capture_index); + n++; + } } } capture_index++; @@ -1699,17 +1733,18 @@ static int re_parse_captures(REParseState *s, int *phas_named_captures, } } done: - if (capture_name) - return -1; - else + if (capture_name) { + return n; + } else { return capture_index; + } } static int re_count_captures(REParseState *s) { if (s->total_capture_count < 0) { s->total_capture_count = re_parse_captures(s, &s->has_named_captures, - NULL); + NULL, FALSE); } return s->total_capture_count; } @@ -1721,25 +1756,53 @@ static BOOL re_has_named_captures(REParseState *s) return s->has_named_captures; } -static int find_group_name(REParseState *s, const char *name) +static int find_group_name(REParseState *s, const char *name, BOOL emit_group_index) { const char *p, *buf_end; size_t len, name_len; - int capture_index; + int capture_index, n; p = (char *)s->group_names.buf; - if (!p) return -1; + if (!p) + return 0; buf_end = (char *)s->group_names.buf + s->group_names.size; name_len = strlen(name); capture_index = 1; + n = 0; while (p < buf_end) { len = strlen(p); - if (len == name_len && memcmp(name, p, name_len) == 0) - return capture_index; - p += len + 1; + if (len == name_len && memcmp(name, p, name_len) == 0) { + if (emit_group_index) + dbuf_putc(&s->byte_code, capture_index); + n++; + } + p += len + LRE_GROUP_NAME_TRAILER_LEN; capture_index++; } - return -1; + return n; +} + +static BOOL is_duplicate_group_name(REParseState *s, const char *name, int scope) +{ + const char *p, *buf_end; + size_t len, name_len; + int scope1; + + p = (char *)s->group_names.buf; + if (!p) + return 0; + buf_end = (char *)s->group_names.buf + s->group_names.size; + name_len = strlen(name); + while (p < buf_end) { + len = strlen(p); + if (len == name_len && memcmp(name, p, name_len) == 0) { + scope1 = (uint8_t)p[len + 1]; + if (scope == scope1) + return TRUE; + } + p += len + LRE_GROUP_NAME_TRAILER_LEN; + } + return FALSE; } static int re_parse_disjunction(REParseState *s, BOOL is_backward_dir); @@ -1783,7 +1846,7 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) { const uint8_t *p; int c, last_atom_start, quant_min, quant_max, last_capture_count; - BOOL greedy, add_zero_advance_check, is_neg, is_backward_lookahead; + BOOL greedy, is_neg, is_backward_lookahead; REStringList cr_s, *cr = &cr_s; last_atom_start = -1; @@ -1922,12 +1985,16 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) &p)) { return re_parse_error(s, "invalid group name"); } - if (find_group_name(s, s->u.tmp_buf) > 0) { + /* poor's man method to test duplicate group + names. */ + /* XXX: this method does not catch all the errors*/ + if (is_duplicate_group_name(s, s->u.tmp_buf, s->group_name_scope)) { return re_parse_error(s, "duplicate group name"); } /* group name with a trailing zero */ dbuf_put(&s->group_names, (uint8_t *)s->u.tmp_buf, strlen(s->u.tmp_buf) + 1); + dbuf_putc(&s->group_names, s->group_name_scope); s->has_named_captures = 1; goto parse_capture; } else { @@ -1938,6 +2005,7 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) p++; /* capture without group name */ dbuf_putc(&s->group_names, 0); + dbuf_putc(&s->group_names, 0); parse_capture: if (s->capture_count >= CAPTURE_COUNT_MAX) return re_parse_error(s, "too many captures"); @@ -1964,17 +2032,18 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) case 'b': case 'B': if (p[1] != 'b') { - re_emit_op(s, s->ignore_case ? REOP_not_word_boundary_i : REOP_not_word_boundary); + re_emit_op(s, s->ignore_case && s->is_unicode ? REOP_not_word_boundary_i : REOP_not_word_boundary); } else { - re_emit_op(s, s->ignore_case ? REOP_word_boundary_i : REOP_word_boundary); + re_emit_op(s, s->ignore_case && s->is_unicode ? REOP_word_boundary_i : REOP_word_boundary); } p += 2; break; case 'k': { const uint8_t *p1; - int dummy_res; - + int dummy_res, n; + BOOL is_forward; + p1 = p; if (p1[2] != '<') { /* annex B: we tolerate invalid group names in non @@ -1993,21 +2062,33 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) else goto parse_class_atom; } - c = find_group_name(s, s->u.tmp_buf); - if (c < 0) { + is_forward = FALSE; + n = find_group_name(s, s->u.tmp_buf, FALSE); + if (n == 0) { /* no capture name parsed before, try to look after (inefficient, but hopefully not common */ - c = re_parse_captures(s, &dummy_res, s->u.tmp_buf); - if (c < 0) { + n = re_parse_captures(s, &dummy_res, s->u.tmp_buf, FALSE); + if (n == 0) { if (s->is_unicode || re_has_named_captures(s)) return re_parse_error(s, "group name not defined"); else goto parse_class_atom; } + is_forward = TRUE; + } + last_atom_start = s->byte_code.size; + last_capture_count = s->capture_count; + + /* emit back references to all the captures indexes matching the group name */ + re_emit_op_u8(s, REOP_back_reference + 2 * is_backward_dir + s->ignore_case, n); + if (is_forward) { + re_parse_captures(s, &dummy_res, s->u.tmp_buf, TRUE); + } else { + find_group_name(s, s->u.tmp_buf, TRUE); } p = p1; } - goto emit_back_reference; + break; case '0': p += 2; c = 0; @@ -2053,11 +2134,11 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) } return re_parse_error(s, "back reference out of range in regular expression"); } - emit_back_reference: last_atom_start = s->byte_code.size; last_capture_count = s->capture_count; - re_emit_op_u8(s, REOP_back_reference + 2 * is_backward_dir + s->ignore_case, c); + re_emit_op_u8(s, REOP_back_reference + 2 * is_backward_dir + s->ignore_case, 1); + dbuf_putc(&s->byte_code, c); } break; default: @@ -2090,8 +2171,15 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) if (is_backward_dir) re_emit_op(s, REOP_prev); if (c >= CLASS_RANGE_BASE) { - int ret; - ret = re_emit_string_list(s, cr); + int ret = 0; + /* optimize the common 'space' tests */ + if (c == (CLASS_RANGE_BASE + CHAR_RANGE_s)) { + re_emit_op(s, REOP_space); + } else if (c == (CLASS_RANGE_BASE + CHAR_RANGE_S)) { + re_emit_op(s, REOP_not_space); + } else { + ret = re_emit_string_list(s, cr); + } re_string_list_free(cr); if (ret) return -1; @@ -2166,20 +2254,39 @@ static int re_parse_term(REParseState *s, BOOL is_backward_dir) if (last_atom_start < 0) { return re_parse_error(s, "nothing to repeat"); } - /* the spec tells that if there is no advance when - running the atom after the first quant_min times, - then there is no match. We remove this test when we - are sure the atom always advances the position. */ - add_zero_advance_check = re_need_check_advance(s->byte_code.buf + last_atom_start, - s->byte_code.size - last_atom_start); - { + BOOL need_capture_init, add_zero_advance_check; int len, pos; + + /* the spec tells that if there is no advance when + running the atom after the first quant_min times, + then there is no match. We remove this test when we + are sure the atom always advances the position. */ + add_zero_advance_check = + re_need_check_adv_and_capture_init(&need_capture_init, + s->byte_code.buf + last_atom_start, + s->byte_code.size - last_atom_start); + + /* general case: need to reset the capture at each + iteration. We don't do it if there are no captures + in the atom or if we are sure all captures are + initialized in the atom. If quant_min = 0, we still + need to reset once the captures in case the atom + does not match. */ + if (need_capture_init && last_capture_count != s->capture_count) { + if (dbuf_insert(&s->byte_code, last_atom_start, 3)) + goto out_of_memory; + int pos = last_atom_start; + s->byte_code.buf[pos++] = REOP_save_reset; + s->byte_code.buf[pos++] = last_capture_count; + s->byte_code.buf[pos++] = s->capture_count - 1; + } + len = s->byte_code.size - last_atom_start; if (quant_min == 0) { /* need to reset the capture in case the atom is not executed */ - if (last_capture_count != s->capture_count) { + if (!need_capture_init && last_capture_count != s->capture_count) { if (dbuf_insert(&s->byte_code, last_atom_start, 3)) goto out_of_memory; s->byte_code.buf[last_atom_start++] = REOP_save_reset; @@ -2320,6 +2427,8 @@ static int re_parse_disjunction(REParseState *s, BOOL is_backward_dir) pos = re_emit_op_u32(s, REOP_goto, 0); + s->group_name_scope++; + if (re_parse_alternative(s, is_backward_dir)) return -1; @@ -2382,6 +2491,13 @@ static int compute_register_count(uint8_t *bc_buf, int bc_buf_len) val = get_u16(bc_buf + pos + 1); len += val * 8; break; + case REOP_back_reference: + case REOP_back_reference_i: + case REOP_backward_back_reference: + case REOP_backward_back_reference_i: + val = bc_buf[pos + 1]; + len += val; + break; } pos += len; } @@ -2481,7 +2597,7 @@ uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size, s->byte_code.size - RE_HEADER_LEN); /* add the named groups if needed */ - if (s->group_names.size > (s->capture_count - 1)) { + if (s->group_names.size > (s->capture_count - 1) * LRE_GROUP_NAME_TRAILER_LEN) { dbuf_put(&s->byte_code, s->group_names.buf, s->group_names.size); put_u16(s->byte_code.buf + RE_HEADER_FLAGS, lre_get_flags(s->byte_code.buf) | LRE_FLAG_NAMED_GROUPS); @@ -2502,14 +2618,6 @@ static BOOL is_line_terminator(uint32_t c) return (c == '\n' || c == '\r' || c == CP_LS || c == CP_PS); } -static BOOL is_word_char(uint32_t c) -{ - return ((c >= '0' && c <= '9') || - (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c == '_')); -} - #define GET_CHAR(c, cptr, cbuf_end, cbuf_type) \ do { \ if (cbuf_type == 0) { \ @@ -2664,7 +2772,7 @@ static no_inline int stack_realloc(REExecContext *s, size_t n) /* return 1 if match, 0 if not match or < 0 if error. */ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, - uint8_t **regs, const uint8_t *pc, const uint8_t *cptr) + const uint8_t *pc, const uint8_t *cptr) { int opcode; int cbuf_type; @@ -2704,24 +2812,24 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } /* avoid saving the previous value if already saved */ -#define SAVE_REG(idx, value) \ +#define SAVE_CAPTURE_CHECK(idx, value) \ { \ StackElem *sp1; \ sp1 = sp; \ for(;;) { \ if (sp1 > bp) { \ - if (sp1[-2].val == -(int)(idx + 1)) \ + if (sp1[-2].val == idx) \ break; \ sp1 -= 2; \ } else { \ CHECK_STACK_SPACE(2); \ - sp[0].val = -(int)(idx + 1); \ - sp[1].ptr = regs[idx]; \ + sp[0].val = idx; \ + sp[1].ptr = capture[idx]; \ sp += 2; \ break; \ } \ } \ - regs[idx] = (value); \ + capture[idx] = (value); \ } @@ -2746,13 +2854,9 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, REExecStateEnum type; if (bp == s->stack_buf) return 0; - /* undo the modifications to capture[] and regs[] */ + /* undo the modifications to capture[] */ while (sp > bp) { - intptr_t idx2 = sp[-2].val; - if (idx2 >= 0) - capture[idx2] = sp[-1].ptr; - else - regs[-idx2 - 1] = sp[-1].ptr; + capture[sp[-2].val] = sp[-1].ptr; sp -= 2; } @@ -2805,13 +2909,9 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, for(;;) { REExecStateEnum type; type = bp[-1].bp.type; - /* undo the modifications to capture[] and regs[] */ + /* undo the modifications to capture[] */ while (sp > bp) { - intptr_t idx2 = sp[-2].val; - if (idx2 >= 0) - capture[idx2] = sp[-1].ptr; - else - regs[-idx2 - 1] = sp[-1].ptr; + capture[sp[-2].val] = sp[-1].ptr; sp -= 2; } pc = sp[-3].ptr; @@ -2914,6 +3014,20 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, goto no_match; GET_CHAR(c, cptr, cbuf_end, cbuf_type); break; + case REOP_space: + if (cptr == cbuf_end) + goto no_match; + GET_CHAR(c, cptr, cbuf_end, cbuf_type); + if (!lre_is_space(c)) + goto no_match; + break; + case REOP_not_space: + if (cptr == cbuf_end) + goto no_match; + GET_CHAR(c, cptr, cbuf_end, cbuf_type); + if (lre_is_space(c)) + goto no_match; + break; case REOP_save_start: case REOP_save_end: val = *pc++; @@ -2939,20 +3053,20 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } break; case REOP_set_i32: - idx = pc[0]; + idx = 2 * s->capture_count + pc[0]; val = get_u32(pc + 1); pc += 5; - SAVE_REG(idx, (void *)(uintptr_t)val); + SAVE_CAPTURE_CHECK(idx, (void *)(uintptr_t)val); break; case REOP_loop: { uint32_t val2; - idx = pc[0]; + idx = 2 * s->capture_count + pc[0]; val = get_u32(pc + 1); pc += 5; - val2 = (uintptr_t)regs[idx] - 1; - SAVE_REG(idx, (void *)(uintptr_t)val2); + val2 = (uintptr_t)capture[idx] - 1; + SAVE_CAPTURE_CHECK(idx, (void *)(uintptr_t)val2); if (val2 != 0) { pc += (int)val; if (lre_poll_timeout(s)) @@ -2967,14 +3081,14 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, { const uint8_t *pc1; uint32_t val2, limit; - idx = pc[0]; + idx = 2 * s->capture_count + pc[0]; limit = get_u32(pc + 1); val = get_u32(pc + 5); pc += 9; /* decrement the counter */ - val2 = (uintptr_t)regs[idx] - 1; - SAVE_REG(idx, (void *)(uintptr_t)val2); + val2 = (uintptr_t)capture[idx] - 1; + SAVE_CAPTURE_CHECK(idx, (void *)(uintptr_t)val2); if (val2 > limit) { /* normal loop if counter > limit */ @@ -2985,7 +3099,7 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, /* check advance */ if ((opcode == REOP_loop_check_adv_split_goto_first || opcode == REOP_loop_check_adv_split_next_first) && - regs[idx + 1] == cptr && + capture[idx + 1] == cptr && val2 != limit) { goto no_match; } @@ -3011,14 +3125,14 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, } break; case REOP_set_char_pos: - idx = pc[0]; + idx = 2 * s->capture_count + pc[0]; pc++; - SAVE_REG(idx, (uint8_t *)cptr); + SAVE_CAPTURE_CHECK(idx, (uint8_t *)cptr); break; case REOP_check_advance: - idx = pc[0]; + idx = 2 * s->capture_count + pc[0]; pc++; - if (regs[idx] == cptr) + if (capture[idx] == cptr) goto no_match; break; case REOP_word_boundary: @@ -3034,18 +3148,22 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, v1 = FALSE; } else { PEEK_PREV_CHAR(c, cptr, s->cbuf, cbuf_type); - if (ignore_case) - c = lre_canonicalize(c, s->is_unicode); - v1 = is_word_char(c); + if (c < 256) { + v1 = (lre_is_word_byte(c) != 0); + } else { + v1 = ignore_case && (c == 0x017f || c == 0x212a); + } } /* current char */ if (cptr >= cbuf_end) { v2 = FALSE; } else { PEEK_CHAR(c, cptr, cbuf_end, cbuf_type); - if (ignore_case) - c = lre_canonicalize(c, s->is_unicode); - v2 = is_word_char(c); + if (c < 256) { + v2 = (lre_is_word_byte(c) != 0); + } else { + v2 = ignore_case && (c == 0x017f || c == 0x212a); + } } if (v1 ^ v2 ^ is_boundary) goto no_match; @@ -3057,43 +3175,53 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture, case REOP_backward_back_reference_i: { const uint8_t *cptr1, *cptr1_end, *cptr1_start; + const uint8_t *pc1; uint32_t c1, c2; + int i, n; - val = *pc++; - if (val >= s->capture_count) - goto no_match; - cptr1_start = capture[2 * val]; - cptr1_end = capture[2 * val + 1]; - if (!cptr1_start || !cptr1_end) - break; - if (opcode == REOP_back_reference || - opcode == REOP_back_reference_i) { - cptr1 = cptr1_start; - while (cptr1 < cptr1_end) { - if (cptr >= cbuf_end) - goto no_match; - GET_CHAR(c1, cptr1, cptr1_end, cbuf_type); - GET_CHAR(c2, cptr, cbuf_end, cbuf_type); - if (opcode == REOP_back_reference_i) { - c1 = lre_canonicalize(c1, s->is_unicode); - c2 = lre_canonicalize(c2, s->is_unicode); - } - if (c1 != c2) - goto no_match; - } - } else { - cptr1 = cptr1_end; - while (cptr1 > cptr1_start) { - if (cptr == s->cbuf) - goto no_match; - GET_PREV_CHAR(c1, cptr1, cptr1_start, cbuf_type); - GET_PREV_CHAR(c2, cptr, s->cbuf, cbuf_type); - if (opcode == REOP_backward_back_reference_i) { - c1 = lre_canonicalize(c1, s->is_unicode); - c2 = lre_canonicalize(c2, s->is_unicode); + n = *pc++; + pc1 = pc; + pc += n; + + for(i = 0; i < n; i++) { + val = pc1[i]; + if (val >= s->capture_count) + goto no_match; + cptr1_start = capture[2 * val]; + cptr1_end = capture[2 * val + 1]; + /* test the first not empty capture */ + if (cptr1_start && cptr1_end) { + if (opcode == REOP_back_reference || + opcode == REOP_back_reference_i) { + cptr1 = cptr1_start; + while (cptr1 < cptr1_end) { + if (cptr >= cbuf_end) + goto no_match; + GET_CHAR(c1, cptr1, cptr1_end, cbuf_type); + GET_CHAR(c2, cptr, cbuf_end, cbuf_type); + if (opcode == REOP_back_reference_i) { + c1 = lre_canonicalize(c1, s->is_unicode); + c2 = lre_canonicalize(c2, s->is_unicode); + } + if (c1 != c2) + goto no_match; + } + } else { + cptr1 = cptr1_end; + while (cptr1 > cptr1_start) { + if (cptr == s->cbuf) + goto no_match; + GET_PREV_CHAR(c1, cptr1, cptr1_start, cbuf_type); + GET_PREV_CHAR(c2, cptr, s->cbuf, cbuf_type); + if (opcode == REOP_backward_back_reference_i) { + c1 = lre_canonicalize(c1, s->is_unicode); + c2 = lre_canonicalize(c2, s->is_unicode); + } + if (c1 != c2) + goto no_match; + } } - if (c1 != c2) - goto no_match; + break; } } } @@ -3200,8 +3328,7 @@ int lre_exec(uint8_t **capture, int cbuf_type, void *opaque) { REExecContext s_s, *s = &s_s; - int re_flags, i, ret, register_count; - uint8_t **regs; + int re_flags, i, ret; const uint8_t *cptr; re_flags = lre_get_flags(bc_buf); @@ -3220,10 +3347,6 @@ int lre_exec(uint8_t **capture, for(i = 0; i < s->capture_count * 2; i++) capture[i] = NULL; - /* XXX: modify the API so that the registers are allocated after - the captures to suppress some tests */ - register_count = bc_buf[RE_HEADER_REGISTER_COUNT]; - regs = alloca(register_count * sizeof(regs[0])); cptr = cbuf + (cindex << cbuf_type); if (0 < cindex && cindex < clen && s->cbuf_type == 2) { @@ -3233,13 +3356,19 @@ int lre_exec(uint8_t **capture, } } - ret = lre_exec_backtrack(s, capture, regs, bc_buf + RE_HEADER_LEN, - cptr); + ret = lre_exec_backtrack(s, capture, bc_buf + RE_HEADER_LEN, cptr); + if (s->stack_buf != s->static_stack_buf) lre_realloc(s->opaque, s->stack_buf, 0); return ret; } +int lre_get_alloc_count(const uint8_t *bc_buf) +{ + return bc_buf[RE_HEADER_CAPTURE_COUNT] * 2 + + bc_buf[RE_HEADER_REGISTER_COUNT]; +} + int lre_get_capture_count(const uint8_t *bc_buf) { return bc_buf[RE_HEADER_CAPTURE_COUNT]; @@ -3278,7 +3407,7 @@ int main(int argc, char **argv) int len, flags, ret, i; uint8_t *bc; char error_msg[64]; - uint8_t *capture[CAPTURE_COUNT_MAX * 2]; + uint8_t *capture; const char *input; int input_len, capture_count; @@ -3297,6 +3426,7 @@ int main(int argc, char **argv) input = argv[3]; input_len = strlen(input); + capture = malloc(sizeof(capture[0]) * lre_get_alloc_count(bc)); ret = lre_exec(capture, bc, (uint8_t *)input, 0, input_len, 0, NULL); printf("ret=%d\n", ret); if (ret == 1) { @@ -3312,6 +3442,7 @@ int main(int argc, char **argv) printf("\n"); } } + free(capture); return 0; } #endif diff --git a/src/couch_quickjs/quickjs/libregexp.h b/src/couch_quickjs/quickjs/libregexp.h index da76e4cef61..0905bcb7973 100644 --- a/src/couch_quickjs/quickjs/libregexp.h +++ b/src/couch_quickjs/quickjs/libregexp.h @@ -40,9 +40,13 @@ #define LRE_RET_MEMORY_ERROR (-1) #define LRE_RET_TIMEOUT (-2) +/* trailer length after the group name including the trailing '\0' */ +#define LRE_GROUP_NAME_TRAILER_LEN 2 + uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size, const char *buf, size_t buf_len, int re_flags, void *opaque); +int lre_get_alloc_count(const uint8_t *bc_buf); int lre_get_capture_count(const uint8_t *bc_buf); int lre_get_flags(const uint8_t *bc_buf); const char *lre_get_groupnames(const uint8_t *bc_buf); diff --git a/src/couch_quickjs/quickjs/libunicode.h b/src/couch_quickjs/quickjs/libunicode.h index 5d964e40f73..5b02c82b441 100644 --- a/src/couch_quickjs/quickjs/libunicode.h +++ b/src/couch_quickjs/quickjs/libunicode.h @@ -147,6 +147,11 @@ static inline int lre_is_id_continue_byte(uint8_t c) { UNICODE_C_DIGIT); } +static inline int lre_is_word_byte(uint8_t c) { + return lre_ctype_bits[c] & (UNICODE_C_UPPER | UNICODE_C_LOWER | + UNICODE_C_UNDER | UNICODE_C_DIGIT); +} + int lre_is_space_non_ascii(uint32_t c); static inline int lre_is_space(uint32_t c) { diff --git a/src/couch_quickjs/quickjs/quickjs-opcode.h b/src/couch_quickjs/quickjs/quickjs-opcode.h index d93852133df..7b98ddf055e 100644 --- a/src/couch_quickjs/quickjs/quickjs-opcode.h +++ b/src/couch_quickjs/quickjs/quickjs-opcode.h @@ -168,6 +168,7 @@ DEF( set_var_ref, 3, 1, 1, var_ref) /* must come after put_var_ref */ DEF(set_loc_uninitialized, 3, 0, 0, loc) DEF( get_loc_check, 3, 0, 1, loc) DEF( put_loc_check, 3, 1, 0, loc) /* must come after get_loc_check */ +DEF( set_loc_check, 3, 1, 1, loc) /* must come after put_loc_check */ DEF( put_loc_check_init, 3, 1, 0, loc) DEF(get_loc_checkthis, 3, 0, 1, loc) DEF(get_var_ref_check, 3, 0, 1, var_ref) diff --git a/src/couch_quickjs/quickjs/quickjs.c b/src/couch_quickjs/quickjs/quickjs.c index b84af4a27ad..6cad52d76c5 100644 --- a/src/couch_quickjs/quickjs/quickjs.c +++ b/src/couch_quickjs/quickjs/quickjs.c @@ -451,13 +451,6 @@ struct JSContext { uint16_t binary_object_count; int binary_object_size; - /* TRUE if the array prototype is "normal": - - no small index properties which are get/set or non writable - - its prototype is Object.prototype - - Object.prototype has no small index properties which are get/set or non writable - - the prototype of Object.prototype is null (always true as it is immutable) - */ - uint8_t std_array_prototype; JSShape *array_shape; /* initial shape for Array objects */ JSShape *arguments_shape; /* shape for arguments objects */ @@ -936,7 +929,13 @@ struct JSObject { struct { int __gc_ref_count; /* corresponds to header.ref_count */ uint8_t __gc_mark : 7; /* corresponds to header.mark/gc_obj_type */ - uint8_t is_prototype : 1; /* object may be used as prototype */ + /* TRUE if the array prototype is "normal": + - no small index properties which are get/set or non writable + - its prototype is Object.prototype + - Object.prototype has no small index properties which are get/set or non writable + - the prototype of Object.prototype is null (always true as it is immutable) + */ + uint8_t is_std_array_prototype : 1; uint8_t extensible : 1; uint8_t free_mark : 1; /* only used when freeing objects with cycles */ @@ -5206,7 +5205,7 @@ static JSValue JS_NewObjectFromShape(JSContext *ctx, JSShape *sh, JSClassID clas if (unlikely(!p)) goto fail; p->class_id = class_id; - p->is_prototype = 0; + p->is_std_array_prototype = 0; p->extensible = TRUE; p->free_mark = 0; p->is_exotic = 0; @@ -7566,14 +7565,7 @@ static int JS_SetPrototypeInternal(JSContext *ctx, JSValueConst obj, if (sh->proto) JS_FreeValue(ctx, JS_MKPTR(JS_TAG_OBJECT, sh->proto)); sh->proto = proto; - if (proto) - proto->is_prototype = TRUE; - if (p->is_prototype) { - /* track modification of Array.prototype */ - if (unlikely(p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]))) { - ctx->std_array_prototype = FALSE; - } - } + p->is_std_array_prototype = FALSE; return TRUE; } @@ -8773,12 +8765,25 @@ static JSProperty *add_property(JSContext *ctx, { JSShape *sh, *new_sh; - if (unlikely(p->is_prototype)) { - /* track addition of small integer properties to Array.prototype and Object.prototype */ - if (unlikely((p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || - p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_OBJECT])) && - __JS_AtomIsTaggedInt(prop))) { - ctx->std_array_prototype = FALSE; + if (unlikely(__JS_AtomIsTaggedInt(prop))) { + /* update is_std_array_prototype */ + if (unlikely(p->is_std_array_prototype)) { + p->is_std_array_prototype = FALSE; + } else if (unlikely(p->has_immutable_prototype)) { + struct list_head *el; + + /* modifying Object.prototype : reset the corresponding is_std_array_prototype */ + list_for_each(el, &ctx->rt->context_list) { + JSContext *ctx1 = list_entry(el, JSContext, link); + if (JS_IsObject(ctx1->class_proto[JS_CLASS_OBJECT]) && + JS_VALUE_GET_OBJ(ctx1->class_proto[JS_CLASS_OBJECT]) == p) { + if (JS_IsObject(ctx1->class_proto[JS_CLASS_ARRAY])) { + JSObject *p1 = JS_VALUE_GET_OBJ(ctx1->class_proto[JS_CLASS_ARRAY]); + p1->is_std_array_prototype = FALSE; + } + break; + } + } } } sh = p->shape; @@ -8860,11 +8865,7 @@ static no_inline __exception int convert_fast_array_to_array(JSContext *ctx, p->u.array.u.values = NULL; /* fail safe */ p->u.array.u1.size = 0; p->fast_array = 0; - - /* track modification of Array.prototype */ - if (unlikely(p == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]))) { - ctx->std_array_prototype = FALSE; - } + p->is_std_array_prototype = FALSE; return 0; } @@ -9509,6 +9510,18 @@ int JS_SetPropertyInternal(JSContext *ctx, JSValueConst obj, } } +/* return true if an element can be added to a fast array without further tests */ +static force_inline BOOL can_extend_fast_array(JSObject *p) +{ + JSObject *proto; + if (!p->extensible) + return FALSE; + proto = p->shape->proto; + if (!proto) + return TRUE; + return proto->is_std_array_prototype; +} + /* flags can be JS_PROP_THROW or JS_PROP_THROW_STRICT */ static int JS_SetPropertyValue(JSContext *ctx, JSValueConst this_obj, JSValue prop, JSValue val, int flags) @@ -9529,9 +9542,7 @@ static int JS_SetPropertyValue(JSContext *ctx, JSValueConst this_obj, /* fast path to add an element to the array */ if (unlikely(idx != (uint32_t)p->u.array.count || !p->fast_array || - !p->extensible || - p->shape->proto != JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || - !ctx->std_array_prototype)) { + !can_extend_fast_array(p))) { goto slow_path; } /* add element */ @@ -18335,6 +18346,18 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, sp--; } BREAK; + CASE(OP_set_loc_check): + { + int idx; + idx = get_u16(pc); + pc += 2; + if (unlikely(JS_IsUninitialized(var_buf[idx]))) { + JS_ThrowReferenceErrorUninitialized2(ctx, b, idx, FALSE); + goto exception; + } + set_value(ctx, &var_buf[idx], JS_DupValue(ctx, sp[-1])); + } + BREAK; CASE(OP_put_loc_check_init): { int idx; @@ -19142,9 +19165,7 @@ static JSValue JS_CallInternal(JSContext *caller_ctx, JSValueConst func_obj, uint32_t new_len, array_len; if (unlikely(idx != (uint32_t)p->u.array.count || !p->fast_array || - !p->extensible || - p->shape->proto != JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) || - !ctx->std_array_prototype)) { + !can_extend_fast_array(p))) { goto put_array_el_slow_path; } if (likely(JS_VALUE_GET_TAG(p->prop[0].u.value) != JS_TAG_INT)) @@ -23355,8 +23376,10 @@ static int cpool_add(JSParseState *s, JSValue val) JSFunctionDef *fd = s->cur_func; if (js_resize_array(s->ctx, (void *)&fd->cpool, sizeof(fd->cpool[0]), - &fd->cpool_size, fd->cpool_count + 1)) + &fd->cpool_size, fd->cpool_count + 1)) { + JS_FreeValue(s->ctx, val); return -1; + } fd->cpool[fd->cpool_count++] = val; return fd->cpool_count - 1; } @@ -29946,6 +29969,7 @@ static int js_create_module_bytecode_function(JSContext *ctx, JSModuleDef *m) if (JS_IsException(func_obj)) return -1; + m->func_obj = func_obj; b = JS_VALUE_GET_PTR(bfunc); func_obj = js_closure2(ctx, func_obj, b, NULL, NULL, TRUE, m); if (JS_IsException(func_obj)) { @@ -29953,7 +29977,6 @@ static int js_create_module_bytecode_function(JSContext *ctx, JSModuleDef *m) JS_FreeValue(ctx, func_obj); return -1; } - m->func_obj = func_obj; return 0; } @@ -34760,7 +34783,7 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) /* Transformation: dup put_x(n) drop -> put_x(n) */ int op1, line2 = -1; /* Transformation: dup put_x(n) -> set_x(n) */ - if (code_match(&cc, pos_next, M3(OP_put_loc, OP_put_arg, OP_put_var_ref), -1, -1)) { + if (code_match(&cc, pos_next, M4(OP_put_loc, OP_put_loc_check, OP_put_arg, OP_put_var_ref), -1, -1)) { if (cc.line_num >= 0) line_num = cc.line_num; op1 = cc.op + 1; /* put_x -> set_x */ pos_next = cc.pos; @@ -34868,6 +34891,7 @@ static __exception int resolve_labels(JSContext *ctx, JSFunctionDef *s) goto no_change; #endif case OP_put_loc: + case OP_put_loc_check: case OP_put_arg: case OP_put_var_ref: if (OPTIMIZE) { @@ -42131,9 +42155,7 @@ static JSValue js_array_push(JSContext *ctx, JSValueConst this_val, if (likely(JS_VALUE_GET_TAG(this_val) == JS_TAG_OBJECT && !unshift)) { JSObject *p = JS_VALUE_GET_OBJ(this_val); if (likely(p->class_id == JS_CLASS_ARRAY && p->fast_array && - p->extensible && - p->shape->proto == JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]) && - ctx->std_array_prototype && + can_extend_fast_array(p) && JS_VALUE_GET_TAG(p->prop[0].u.value) == JS_TAG_INT && JS_VALUE_GET_INT(p->prop[0].u.value) == p->u.array.count && (get_shape_prop(p->shape)->flags & JS_PROP_WRITABLE) != 0)) { @@ -45147,7 +45169,7 @@ static JSValue js_string_match(JSContext *ctx, JSValueConst this_val, if (JS_IsUndefined(O) || JS_IsNull(O)) return JS_ThrowTypeError(ctx, "cannot convert to object"); - if (!JS_IsUndefined(regexp) && !JS_IsNull(regexp)) { + if (JS_IsObject(regexp)) { matcher = JS_GetProperty(ctx, regexp, atom); if (JS_IsException(matcher)) return JS_EXCEPTION; @@ -45327,7 +45349,7 @@ static JSValue js_string_replace(JSContext *ctx, JSValueConst this_val, replaceValue_str = JS_UNDEFINED; repl_str = JS_UNDEFINED; - if (!JS_IsUndefined(searchValue) && !JS_IsNull(searchValue)) { + if (JS_IsObject(searchValue)) { JSValue replacer; if (is_replaceAll) { if (check_regexp_g_flag(ctx, searchValue) < 0) @@ -45438,7 +45460,7 @@ static JSValue js_string_split(JSContext *ctx, JSValueConst this_val, A = JS_UNDEFINED; R = JS_UNDEFINED; - if (!JS_IsUndefined(separator) && !JS_IsNull(separator)) { + if (JS_IsObject(separator)) { JSValue splitter; splitter = JS_GetProperty(ctx, separator, JS_ATOM_Symbol_split); if (JS_IsException(splitter)) @@ -45479,7 +45501,6 @@ static JSValue js_string_split(JSContext *ctx, JSValueConst this_val, goto add_tail; goto done; } - q = p; for (q = p; (q += !r) <= s - r - !r; q = p = e + r) { e = string_indexof(sp, rp, q); if (e < 0) @@ -47415,11 +47436,12 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, JSValue indices, indices_groups; uint8_t *re_bytecode; uint8_t **capture, *str_buf; - int rc, capture_count, shift, i, re_flags; + int rc, capture_count, shift, i, re_flags, alloc_count; int64_t last_index; const char *group_name_ptr; JSObject *p_obj; - + JSAtom group_name; + if (!re) return JS_EXCEPTION; @@ -47433,7 +47455,8 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, indices = JS_UNDEFINED; indices_groups = JS_UNDEFINED; capture = NULL; - + group_name = JS_ATOM_NULL; + if (js_regexp_get_lastIndex(ctx, &last_index, this_val)) goto fail; @@ -47443,12 +47466,13 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, last_index = 0; } str = JS_VALUE_GET_STRING(str_val); - capture_count = lre_get_capture_count(re_bytecode); - if (capture_count > 0) { - capture = js_malloc(ctx, sizeof(capture[0]) * capture_count * 2); + alloc_count = lre_get_alloc_count(re_bytecode); + if (alloc_count > 0) { + capture = js_malloc(ctx, sizeof(capture[0]) * alloc_count); if (!capture) goto fail; } + capture_count = lre_get_capture_count(re_bytecode); shift = str->is_wide_char; str_buf = str->u.str8; if (last_index > str->len) { @@ -47515,15 +47539,20 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, goto fail; for(i = 0; i < capture_count; i++) { - const char *name = NULL; uint8_t **match = &capture[2 * i]; int start = -1; int end = -1; JSValue val; if (group_name_ptr && i > 0) { - if (*group_name_ptr) name = group_name_ptr; - group_name_ptr += strlen(group_name_ptr) + 1; + if (*group_name_ptr) { + /* XXX: slow, should create a shape when the regexp is + compiled */ + group_name = JS_NewAtom(ctx, group_name_ptr); + if (group_name == JS_ATOM_NULL) + goto fail; + } + group_name_ptr += strlen(group_name_ptr) + LRE_GROUP_NAME_TRAILER_LEN; } if (match[0] && match[1]) { @@ -47550,12 +47579,15 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, goto fail; } } - if (name && !JS_IsUndefined(indices_groups)) { - val = JS_DupValue(ctx, val); - if (JS_DefinePropertyValueStr(ctx, indices_groups, - name, val, prop_flags) < 0) { - JS_FreeValue(ctx, val); - goto fail; + if (group_name != JS_ATOM_NULL) { + /* JS_HasProperty() cannot fail here */ + if (!JS_IsUndefined(val) || + !JS_HasProperty(ctx, indices_groups, group_name)) { + if (JS_DefinePropertyValue(ctx, indices_groups, + group_name, JS_DupValue(ctx, val), prop_flags) < 0) { + JS_FreeValue(ctx, val); + goto fail; + } } } if (JS_DefinePropertyValueUint32(ctx, indices, i, val, @@ -47571,13 +47603,19 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, goto fail; } - if (name) { - if (JS_DefinePropertyValueStr(ctx, groups, name, - JS_DupValue(ctx, val), - prop_flags) < 0) { - JS_FreeValue(ctx, val); - goto fail; + if (group_name != JS_ATOM_NULL) { + /* JS_HasProperty() cannot fail here */ + if (!JS_IsUndefined(val) || + !JS_HasProperty(ctx, groups, group_name)) { + if (JS_DefinePropertyValue(ctx, groups, group_name, + JS_DupValue(ctx, val), + prop_flags) < 0) { + JS_FreeValue(ctx, val); + goto fail; + } } + JS_FreeAtom(ctx, group_name); + group_name = JS_ATOM_NULL; } p_obj->u.array.u.values[p_obj->u.array.count++] = val; } @@ -47598,6 +47636,7 @@ static JSValue js_regexp_exec(JSContext *ctx, JSValueConst this_val, ret = obj; obj = JS_UNDEFINED; fail: + JS_FreeAtom(ctx, group_name); JS_FreeValue(ctx, indices_groups); JS_FreeValue(ctx, indices); JS_FreeValue(ctx, str_val); @@ -47617,7 +47656,7 @@ static JSValue js_regexp_replace(JSContext *ctx, JSValueConst this_val, JSValueC uint8_t *re_bytecode; int ret; uint8_t **capture, *str_buf; - int capture_count, shift, re_flags; + int capture_count, alloc_count, shift, re_flags; int next_src_pos, start, end; int64_t last_index; StringBuffer b_s, *b = &b_s; @@ -47651,12 +47690,13 @@ static JSValue js_regexp_replace(JSContext *ctx, JSValueConst this_val, JSValueC if (js_regexp_get_lastIndex(ctx, &last_index, this_val)) goto fail; } - capture_count = lre_get_capture_count(re_bytecode); - if (capture_count > 0) { - capture = js_malloc(ctx, sizeof(capture[0]) * capture_count * 2); + alloc_count = lre_get_alloc_count(re_bytecode); + if (alloc_count > 0) { + capture = js_malloc(ctx, sizeof(capture[0]) * alloc_count); if (!capture) goto fail; } + capture_count = lre_get_capture_count(re_bytecode); fullUnicode = ((re_flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0); shift = str->is_wide_char; str_buf = str->u.str8; @@ -55405,6 +55445,11 @@ static int JS_AddIntrinsicBasicObjects(JSContext *ctx) return -1; ctx->array_ctor = obj; + { + JSObject *p = JS_VALUE_GET_OBJ(ctx->class_proto[JS_CLASS_ARRAY]); + p->is_std_array_prototype = TRUE; + } + ctx->array_shape = js_new_shape2(ctx, get_proto_obj(ctx->class_proto[JS_CLASS_ARRAY]), JS_PROP_INITIAL_HASH_SIZE, 1); if (!ctx->array_shape) @@ -55412,7 +55457,6 @@ static int JS_AddIntrinsicBasicObjects(JSContext *ctx) if (add_shape_property(ctx, &ctx->array_shape, NULL, JS_ATOM_length, JS_PROP_WRITABLE | JS_PROP_LENGTH)) return -1; - ctx->std_array_prototype = TRUE; ctx->arguments_shape = js_new_shape2(ctx, get_proto_obj(ctx->class_proto[JS_CLASS_OBJECT]), JS_PROP_INITIAL_HASH_SIZE, 3); diff --git a/src/couch_quickjs/quickjs/test262.conf b/src/couch_quickjs/quickjs/test262.conf index fe52a0b088b..aa76e639510 100644 --- a/src/couch_quickjs/quickjs/test262.conf +++ b/src/couch_quickjs/quickjs/test262.conf @@ -176,7 +176,7 @@ Reflect.construct Reflect.set Reflect.setPrototypeOf regexp-dotall -regexp-duplicate-named-groups=skip +regexp-duplicate-named-groups regexp-lookbehind regexp-match-indices regexp-modifiers @@ -255,54 +255,6 @@ test262/test/built-ins/ThrowTypeError/unique-per-realm-function-proto.js #test262/test/built-ins/RegExp/CharacterClassEscapes/ #test262/test/built-ins/RegExp/property-escapes/ -# not yet in official specification -test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-bigint-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-boolean-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-boolean-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-number-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-number-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-string-primitive.js -test262/test/built-ins/String/prototype/match/cstm-matcher-on-string-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-bigint-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-bigint-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-number-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-number-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-string-primitive.js -test262/test/built-ins/String/prototype/matchAll/cstm-matchall-on-string-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-bigint-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-bigint-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-boolean-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-boolean-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-number-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-number-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-string-primitive.js -test262/test/built-ins/String/prototype/replace/cstm-replace-on-string-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-bigint-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-bigint-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-boolean-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-boolean-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-number-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-number-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-string-primitive.js -test262/test/built-ins/String/prototype/replaceAll/cstm-replaceall-on-string-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-bigint-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-bigint-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-boolean-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-boolean-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-number-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-number-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-string-primitive.js -test262/test/built-ins/String/prototype/search/cstm-search-on-string-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-bigint-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-bigint-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-boolean-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-boolean-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-number-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-number-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-string-primitive.js -test262/test/built-ins/String/prototype/split/cstm-split-on-string-primitive.js - #################################### # staging tests diff --git a/src/couch_quickjs/quickjs/test262_errors.txt b/src/couch_quickjs/quickjs/test262_errors.txt index d60814ac4cc..796c296e2af 100644 --- a/src/couch_quickjs/quickjs/test262_errors.txt +++ b/src/couch_quickjs/quickjs/test262_errors.txt @@ -33,12 +33,6 @@ test262/test/staging/sm/Function/function-name-for.js:13: Test262Error: Expected test262/test/staging/sm/Function/implicit-this-in-parameter-expression.js:12: Test262Error: Expected SameValue(«[object Object]», «undefined») to be true test262/test/staging/sm/Function/invalid-parameter-list.js:13: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all test262/test/staging/sm/Function/invalid-parameter-list.js:13: strict mode: Test262Error: Expected a SyntaxError to be thrown but no exception was thrown at all -test262/test/staging/sm/RegExp/regress-613820-1.js:12: Test262Error: Actual [aaa, aa, a] and expected [aa, a, a] should have the same contents. -test262/test/staging/sm/RegExp/regress-613820-1.js:12: strict mode: Test262Error: Actual [aaa, aa, a] and expected [aa, a, a] should have the same contents. -test262/test/staging/sm/RegExp/regress-613820-2.js:12: Test262Error: Actual [foobar, f, o, o, b, a, r] and expected [foobar, undefined, undefined, undefined, b, a, r] should have the same contents. -test262/test/staging/sm/RegExp/regress-613820-2.js:12: strict mode: Test262Error: Actual [foobar, f, o, o, b, a, r] and expected [foobar, undefined, undefined, undefined, b, a, r] should have the same contents. -test262/test/staging/sm/RegExp/regress-613820-3.js:12: Test262Error: Actual [aab, a, undefined, ab] and expected [aa, undefined, a, undefined] should have the same contents. -test262/test/staging/sm/RegExp/regress-613820-3.js:12: strict mode: Test262Error: Actual [aab, a, undefined, ab] and expected [aa, undefined, a, undefined] should have the same contents. test262/test/staging/sm/String/string-upper-lower-mapping.js:16: Test262Error: Expected SameValue(«"꟏"», «"꟎"») to be true test262/test/staging/sm/String/string-upper-lower-mapping.js:16: strict mode: Test262Error: Expected SameValue(«"꟏"», «"꟎"») to be true test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:29: Test262Error: Expected a ExpectedError but got a Error diff --git a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl index 52a252fc9fa..4c8d6b0f629 100644 --- a/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl +++ b/src/couch_quickjs/src/couch_quickjs_scanner_plugin.erl @@ -22,7 +22,6 @@ ddoc/3, shards/2, db_opened/2, - doc_id/3, doc_fdi/3, doc/3, db_closing/2 @@ -152,23 +151,20 @@ db_opened(#st{} = St, Db) -> Step = min(MaxStep, max(1, DocTotal div MaxDocs)), {0, [], St#st{doc_cnt = 0, docs_size = 0, doc_step = Step, docs = []}}. -doc_id(#st{} = St, <>, _Db) -> +doc_fdi(#st{} = St, #full_doc_info{deleted = true}, _Db) -> + % Skip deleted; don't even open the doc body {skip, St}; -doc_id(#st{sid = SId, doc_cnt = C, max_docs = M} = St, _DocId, Db) when C > M -> +doc_fdi(#st{} = St, #full_doc_info{id = <>}, _Db) -> + {skip, St}; +doc_fdi(#st{sid = SId, doc_cnt = C, max_docs = M} = St, #full_doc_info{}, Db) when C > M -> Meta = #{sid => SId, db => Db}, ?INFO("reached max docs ~p", [M], Meta), {stop, St}; -doc_id(#st{doc_cnt = C, doc_step = S} = St, _DocId, _Db) when C rem S /= 0 -> +doc_fdi(#st{doc_cnt = C, doc_step = S} = St, #full_doc_info{}, _Db) when C rem S /= 0 -> {skip, St#st{doc_cnt = C + 1}}; -doc_id(#st{doc_cnt = C} = St, _DocId, _Db) -> +doc_fdi(#st{doc_cnt = C} = St, #full_doc_info{}, _Db) -> {ok, St#st{doc_cnt = C + 1}}. -doc_fdi(#st{} = St, #full_doc_info{deleted = true}, _Db) -> - % Skip deleted; don't even open the doc body - {skip, St}; -doc_fdi(#st{} = St, #full_doc_info{}, _Db) -> - {ok, St}. - doc(#st{} = St, Db, #doc{id = DocId} = Doc) -> #st{sid = SId} = St, JsonDoc = couch_query_servers:json_doc(Doc), diff --git a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl index 7277ab65689..fe8a1a68f6f 100644 --- a/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl +++ b/src/couch_quickjs/test/couch_quickjs_scanner_plugin_tests.erl @@ -92,11 +92,11 @@ t_vdu_filter_map({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -125,11 +125,11 @@ t_filter_map({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -158,11 +158,11 @@ t_map_only({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -191,11 +191,11 @@ t_vdu_only({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -224,11 +224,11 @@ t_non_deterministic_vdu({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -257,11 +257,11 @@ t_filter_only({_, DbName}) -> ?assert(num_calls(shards, 2) >= 1), DbOpenedCount = num_calls(db_opened, 2), ?assert(DbOpenedCount >= 2), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(1, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(1, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assert(num_calls(doc, 3) >= 5), DbClosingCount = num_calls(db_closing, 2), ?assertEqual(DbOpenedCount, DbClosingCount), @@ -314,11 +314,11 @@ t_empty_ddoc({_, DbName}) -> ?assertEqual(1, num_calls(ddoc, ['_', DbName, '_'])), ?assert(num_calls(shards, 2) >= 1), ?assertEqual(0, num_calls(db_opened, 2)), - ?assertEqual(0, num_calls(doc_id, ['_', ?DOC1, '_'])), - ?assertEqual(0, num_calls(doc_id, ['_', ?DOC2, '_'])), - ?assertEqual(0, num_calls(doc_id, ['_', ?DOC3, '_'])), - ?assertEqual(0, num_calls(doc_id, ['_', ?DOC4, '_'])), - ?assertEqual(0, num_calls(doc_id, ['_', ?DOC5, '_'])), + ?assertEqual(0, num_calls(doc_fdi, ['_', fdi_id_match(?DOC1), '_'])), + ?assertEqual(0, num_calls(doc_fdi, ['_', fdi_id_match(?DOC2), '_'])), + ?assertEqual(0, num_calls(doc_fdi, ['_', fdi_id_match(?DOC3), '_'])), + ?assertEqual(0, num_calls(doc_fdi, ['_', fdi_id_match(?DOC4), '_'])), + ?assertEqual(0, num_calls(doc_fdi, ['_', fdi_id_match(?DOC5), '_'])), ?assertEqual(0, num_calls(db_closing, 2)), ?assertEqual(0, couch_stats:sample([couchdb, query_server, process_error_exits])), ?assertEqual(0, couch_stats:sample([couchdb, query_server, process_errors])), @@ -476,6 +476,15 @@ mkdoc(Id, #{} = Body) -> Body1 = Body#{<<"_id">> => Id}, jiffy:decode(jiffy:encode(Body1)). +fdi_id_match(Id) -> + #full_doc_info{ + id = Id, + update_seq = '_', + deleted = '_', + rev_tree = '_', + sizes = '_' + }. + num_calls(Fun, Args) -> meck:num_calls(?PLUGIN, Fun, Args). diff --git a/src/couch_replicator/README.md b/src/couch_replicator/README.md index cd1b7bd9e28..8a631219d2f 100644 --- a/src/couch_replicator/README.md +++ b/src/couch_replicator/README.md @@ -40,7 +40,7 @@ A description of each child: control algorithm to converge on the channel capacity. Implemented using a 16-way sharded ETS table to maintain connection state. The table sharding code is split out to `couch_replicator_rate_limiter_tables` module. The - purpose of the module it to maintain and continually estimate sleep + purpose of the module is to maintain and continually estimate sleep intervals for each connection represented as a `{Method, Url}` pair. The interval is updated accordingly on each call to `failure/1` or `success/1` calls. For a successful request, a client should call `success/1`. Whenever @@ -79,7 +79,7 @@ A description of each child: jobs running less than `replicator.max_jobs` (default 500). So the functions does these operations (actual code paste): - ``` + ```erl Running = running_job_count(), Pending = pending_job_count(), stop_excess_jobs(State, Running), @@ -116,7 +116,7 @@ A description of each child: interesting part is how the scheduler picks which jobs to stop and which ones to start: - * Stopping: When picking jobs to stop the scheduler will pick longest + * Stopping: When picking jobs to stop the scheduler will pick the longest running continuous jobs first. The sorting callback function to get the longest running jobs is unsurprisingly called `longest_running/2`. To pick the longest running jobs it looks at the most recent `started` @@ -143,14 +143,14 @@ A description of each child: on how this algorithm works. The last part is how the scheduler treats jobs which keep crashing. If a - job is started but then crashes then that job is considered unhealthy. The + job is started but then crashes, that job is considered unhealthy. The main idea is to penalize such jobs such that they are forced to wait an exponentially larger amount of time with each consecutive crash. A central part to this algorithm is determining what forms a sequence of consecutive crashes. If a job starts then quickly crashes, and after its next start it crashes again, then that would become a sequence of 2 consecutive crashes. The penalty then would be calculated by `backoff_micros/1` function where - the consecutive crash count would end up as the exponent. However for + the consecutive crash count would end up as the exponent. However, for practical concerns there is also maximum penalty specified and that's the equivalent of 10 consecutive crashes. Timewise it ends up being about 8 hours. That means even a job which keep crashing will still get a chance to @@ -189,10 +189,10 @@ A description of each child: is handling of upgrades from the previous version of the replicator when transient states were written to the documents. Two such states were `triggered` and `error`. Both of those states are removed from the document - then then update proceeds in the regular fashion. `failed` documents are + then update proceeds in the regular fashion. `failed` documents are also ignored here. `failed` is a terminal state which indicates the document was somehow unsuitable to become a replication job (it was malformed or a - duplicate). Otherwise the state update proceeds to `process_updated/2`. + duplicate). Otherwise, the state update proceeds to `process_updated/2`. `process_updated/2` is where replication document updates are parsed and translated to `#rep{}` records. The interesting part here is that the @@ -236,7 +236,7 @@ A description of each child: 1. Filter fetching code has failed. In that case worker returns an error. But because the error could be a transient network error, another worker is started to try again. It could fail and return an error - again, then another one is started and so on. However each consecutive + again, then another one is started and so on. However, each consecutive worker will do an exponential backoff, not unlike the scheduler code. `error_backoff/1` is where the backoff period is calculated. Consecutive errors are held in the `errcnt` field in the ETS table. @@ -255,6 +255,3 @@ A description of each child: cluster, it's ok to check filter changes often. But when there are lots of replications running, having each one checking their filter often is not a good idea. - - - diff --git a/src/couch_replicator/src/couch_replicator_doc_processor.erl b/src/couch_replicator/src/couch_replicator_doc_processor.erl index 533f8d75432..646d5f173f2 100644 --- a/src/couch_replicator/src/couch_replicator_doc_processor.erl +++ b/src/couch_replicator/src/couch_replicator_doc_processor.erl @@ -58,7 +58,7 @@ -define(MIN_START_DELAY_MSEC, 500). -type filter_type() :: nil | view | user | docids | mango. --type repstate() :: initializing | error | scheduled | not_owner. +-type repstate() :: initializing | error | scheduled | not_owner | pending. -record(st, { % Timer reference @@ -167,7 +167,7 @@ process_updated({DbName, _DocId} = Id, JsonRepDoc, Owner = true) -> % Parsing replication doc (but not calculating the id) could throw an % exception which would indicate this document is malformed. This exception % should propagate to db_change function and will be recorded as permanent - % failure in the document. User will have to update the documet to fix the + % failure in the document. User will have to update the document to fix the % problem. Rep0 = couch_replicator_parse:parse_rep_doc_without_id(JsonRepDoc), Rep = Rep0#rep{db_name = DbName, start_time = os:timestamp()}, @@ -563,7 +563,7 @@ doc(Db, DocId) -> end. -spec doc_lookup(binary(), binary(), integer()) -> - {ok, {[_]}} | {error, not_found}. + {ok, {[_]} | nil} | {error, not_found}. doc_lookup(Db, DocId, HealthThreshold) -> case ets:lookup(?MODULE, {Db, DocId}) of [#rdoc{} = RDoc] -> @@ -584,7 +584,13 @@ ejson_doc(#rdoc{state = scheduled} = RDoc, HealthThreshold) -> JobProps = couch_replicator_scheduler:job_summary(RepId, HealthThreshold), case JobProps of nil -> - nil; + % The job is in the doc processor table as "scheduled" but it's not + % in the scheduler's table. This is a transitional state and may + % happen, for example, when jobs complete, but the replicator doc + % processor hasn't yet noticed the "completed" changes feed event + % from the doc update. Since know some info about the job return + % that as pending state instead of just returning nil. + ejson_doc(RDoc#rdoc{state = pending}, HealthThreshold); [{_, _} | _] -> {[ {doc_id, DocId}, @@ -669,8 +675,8 @@ start_scanner(#st{mdb_changes_pid = undefined} = St) -> start_delay_msec() -> DefaultSec = ?DEFAULT_START_DELAY_MSEC div 1000, - % We're using a compatiblity config setting (cluster_start_period) to avoid - % introducting a new config value. + % We're using a compatibility config setting (cluster_start_period) to avoid + % introducing a new config value. MSec = 1000 * config:get_integer("replicator", "cluster_start_period", DefaultSec), max(MSec, ?MIN_START_DELAY_MSEC). diff --git a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl index 5a62b3e06dd..8bb22ac45ef 100644 --- a/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl +++ b/src/couch_replicator/src/couch_replicator_doc_processor_worker.erl @@ -230,7 +230,7 @@ t_ignore_if_doc_deleted(_) -> ?assertEqual(ignore, maybe_start_replication(Id, Rep, make_ref())), ?assertNot(added_job()). -% Should not add job if by the time worker got to fetchign the filter +% Should not add job if by the time worker got to fetch the filter % and building a replication id, another worker was spawned. t_ignore_if_worker_ref_does_not_match(_) -> Id = {?DB, ?DOC1}, diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl index d28ae908cde..8f134aaab8d 100644 --- a/src/couch_replicator/src/couch_replicator_docs.erl +++ b/src/couch_replicator/src/couch_replicator_docs.erl @@ -137,7 +137,7 @@ delete_old_rep_ddoc(RepDb, DDocId) -> ok. % Update a #rep{} record with a replication_id. Calculating the id might involve -% fetching a filter from the source db, and so it could fail intermetently. +% fetching a filter from the source db, and so it could fail intermittently. % In case of a failure to fetch the filter this function will throw a % `{filter_fetch_error, Reason} exception. update_rep_id(Rep) -> diff --git a/src/couch_replicator/src/couch_replicator_filters.erl b/src/couch_replicator/src/couch_replicator_filters.erl index aab8e80b3ed..c6f51d72479 100644 --- a/src/couch_replicator/src/couch_replicator_filters.erl +++ b/src/couch_replicator/src/couch_replicator_filters.erl @@ -60,7 +60,7 @@ parse(Options) -> % Fetches body of filter function from source database. Guaranteed to either % return {ok, Body} or an {error, Reason}. Also assume this function might -% block due to network / socket issues for an undeterminted amount of time. +% block due to network / socket issues for an undetermined amount of time. -spec fetch(binary(), binary(), binary()) -> {ok, {[_]}} | {error, binary()}. fetch(DDocName, FilterName, Source) -> diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl index 0c381b0096e..9916421e75f 100644 --- a/src/couch_replicator/src/couch_replicator_ids.erl +++ b/src/couch_replicator/src/couch_replicator_ids.erl @@ -120,7 +120,15 @@ maybe_append_filters( true -> [<<"winning_revs_only">>] end, - couch_util:to_hex(couch_hash:md5_hash(?term_to_bin(Base3))). + Base4 = + Base3 ++ + case couch_util:get_value(since_seq, Options) of + undefined -> + []; + SinceSeq -> + [SinceSeq] + end, + couch_util:to_hex(couch_hash:md5_hash(?term_to_bin(Base4))). maybe_append_options(Options, RepOptions) -> lists:foldl( diff --git a/src/couch_replicator/src/couch_replicator_parse.erl b/src/couch_replicator/src/couch_replicator_parse.erl index 5f8437992ba..f2d40acbeab 100644 --- a/src/couch_replicator/src/couch_replicator_parse.erl +++ b/src/couch_replicator/src/couch_replicator_parse.erl @@ -63,7 +63,7 @@ default_options() -> % database. If failure or parsing of filter docs fails, parse_doc throws a % {filter_fetch_error, Error} exception. This exception should be considered % transient in respect to the contents of the document itself, since it depends -% on netowrk availability of the source db and other factors. +% on network availability of the source db and other factors. -spec parse_rep_doc({[_]}) -> #rep{}. parse_rep_doc(RepDoc) -> {ok, Rep} = @@ -167,7 +167,7 @@ parse_proxy_settings(Props) when is_list(Props) -> end. % Update a #rep{} record with a replication_id. Calculating the id might involve -% fetching a filter from the source db, and so it could fail intermetently. +% fetching a filter from the source db, and so it could fail intermittently. % In case of a failure to fetch the filter this function will throw a % `{filter_fetch_error, Reason} exception. update_rep_id(Rep) -> diff --git a/src/couch_replicator/src/couch_replicator_rate_limiter.erl b/src/couch_replicator/src/couch_replicator_rate_limiter.erl index ccffd901144..d76052ec7a6 100644 --- a/src/couch_replicator/src/couch_replicator_rate_limiter.erl +++ b/src/couch_replicator/src/couch_replicator_rate_limiter.erl @@ -22,7 +22,7 @@ % % The algorithm referenced above estimates a rate, whereas the implemented % algorithm uses an interval (in milliseconds). It preserves the original -% semantics, that is the failure part is multplicative and the success part is +% semantics, that is the failure part is multiplicative and the success part is % additive. The relationship between rate and interval is: rate = 1000 / % interval. % diff --git a/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl b/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl index 2e255688812..1b79892906e 100644 --- a/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl +++ b/src/couch_replicator/src/couch_replicator_rate_limiter_tables.erl @@ -20,7 +20,7 @@ % in effect (also configurable). % % This module is also in charge of calculating ownership of replications based -% on where their _repicator db documents shards live. +% on where their _replicator db documents shards live. -module(couch_replicator_rate_limiter_tables). diff --git a/src/couch_replicator/src/couch_replicator_scheduler.erl b/src/couch_replicator/src/couch_replicator_scheduler.erl index 14cca4a21a7..9eb0b4723eb 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler.erl @@ -73,7 +73,7 @@ % Worker children get a default 5 second shutdown timeout, so pick a value just % a bit less than that: 4.5 seconds. In couch_replicator_sup our scheduler -% worker doesn't specify the timeout, so it up picks ups the OTP default of 5 +% worker doesn't specify the timeout, so it picks up the OTP default of 5 % seconds https://www.erlang.org/doc/system/sup_princ.html#child-specification % -define(TERMINATE_SHUTDOWN_TIME, 4500). @@ -173,9 +173,9 @@ job_proxy_url(_Endpoint) -> null. % Health threshold is the minimum amount of time an unhealthy job should run -% crashing before it is considered to be healthy again. HealtThreashold should +% crashing before it is considered to be healthy again. Health threshold should % not be 0 as jobs could start and immediately crash, and it shouldn't be -% infinity, since then consecutive crashes would accumulate forever even if +% infinity, since then consecutive crashes would accumulate forever even if % job is back to normal. -spec health_threshold() -> non_neg_integer(). health_threshold() -> @@ -522,7 +522,7 @@ pending_fold(Job, {Set, Now, Count, HealthThreshold}) -> % Replace Job in the accumulator if it has a higher priority (lower priority % value) than the lowest priority there. Job priority is indexed by -% {FairSharePiority, LastStarted} tuples. If the FairSharePriority is the same +% {FairSharePriority, LastStarted} tuples. If the FairSharePriority is the same % then last started timestamp is used to pick. The goal is to keep up to Count % oldest jobs during the iteration. For example, if there are jobs with these % priorities accumulated so far [5, 7, 11], and the priority of current job is @@ -594,14 +594,13 @@ not_recently_crashed(#job{history = History}, Now, HealthThreshold) -> % and running successfully without crashing for a period of time. That period % of time is the HealthThreshold. % - -spec consecutive_crashes(history(), non_neg_integer()) -> non_neg_integer(). consecutive_crashes(History, HealthThreshold) when is_list(History) -> consecutive_crashes(History, HealthThreshold, 0). -spec consecutive_crashes(history(), non_neg_integer(), non_neg_integer()) -> non_neg_integer(). -consecutive_crashes([], _HealthThreashold, Count) -> +consecutive_crashes([], _HealthThreshold, Count) -> Count; consecutive_crashes( [{{crashed, _}, CrashT}, {_, PrevT} = PrevEvent | Rest], @@ -795,7 +794,7 @@ rotate_jobs(State, ChurnSoFar) -> if SlotsAvailable >= 0 -> % If there is are enough SlotsAvailable reduce StopCount to avoid - % unnesessarily stopping jobs. `stop_jobs/3` ignores 0 or negative + % unnecessarily stopping jobs. `stop_jobs/3` ignores 0 or negative % values so we don't worry about that here. StopCount = lists:min([Pending - SlotsAvailable, Running, Churn]), stop_jobs(StopCount, true, State), @@ -930,7 +929,7 @@ optimize_int_option({Key, Val}, #rep{options = Options} = Rep) -> % Updater is a separate process. It receives `update_stats` messages and % updates scheduler stats from the scheduler jobs table. Updates are % performed no more frequently than once per ?STATS_UPDATE_WAIT milliseconds. - +% update_running_jobs_stats(StatsPid) when is_pid(StatsPid) -> StatsPid ! update_stats, ok. diff --git a/src/couch_replicator/src/couch_replicator_scheduler_job.erl b/src/couch_replicator/src/couch_replicator_scheduler_job.erl index f82e3626930..55a28ad7e71 100644 --- a/src/couch_replicator/src/couch_replicator_scheduler_job.erl +++ b/src/couch_replicator/src/couch_replicator_scheduler_job.erl @@ -110,7 +110,7 @@ stop(Pid) when is_pid(Pid) -> % In the rare case the job is already stopping as we try to stop it, it % won't return ok but exit the calling process, usually the scheduler, so % we guard against that. See: - % www.erlang.org/doc/apps/stdlib/gen_server.html#stop/3 + % https://www.erlang.org/doc/apps/stdlib/gen_server.html#stop/3 catch gen_server:stop(Pid, shutdown, ?STOP_TIMEOUT_MSEC), exit(Pid, kill), receive @@ -189,7 +189,7 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx = UserCtx} = Rep) - % Restarting a temporary supervised child implies that the original arguments % (#rep{} record) specified in the MFA component of the supervisor % child spec will always be used whenever the child is restarted. - % This implies the same replication performance tunning parameters will + % This implies the same replication performance tuning parameters will % always be used. The solution is to delete the child spec (see % cancel_replication/1) and then start the replication again, but this is % unfortunately not immune to race conditions. @@ -684,7 +684,16 @@ init_state(Rep) -> end, Stats = couch_replicator_stats:max_stats(ArgStats1, HistoryStats), - StartSeq1 = get_value(since_seq, Options, StartSeq0), + StartSeq1 = + case StartSeq0 of + 0 -> + % Checkpoint doesn't exist, use the `since_seq` to replicate; + % If `since_seq` is not defined, replicate from scratch. + get_value(since_seq, Options, 0); + _ -> + % Replicate with the checkpoint and ignore `since_seq`. + StartSeq0 + end, StartSeq = {0, StartSeq1}, SourceSeq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ), diff --git a/src/couch_replicator/src/couch_replicator_share.erl b/src/couch_replicator/src/couch_replicator_share.erl index fb7b2fcfc56..bf9f2367b5a 100644 --- a/src/couch_replicator/src/couch_replicator_share.erl +++ b/src/couch_replicator/src/couch_replicator_share.erl @@ -39,7 +39,7 @@ % 3) Jobs which run longer accumulate more charges and get assigned a % higher priority value and get to wait longer to run. % -% In order to prevent job starvation, all job priorities are periodicaly +% In order to prevent job starvation, all job priorities are periodically % decayed (decreased). This effectively moves all the jobs towards the front of % the run queue. So, in effect, there are two competing processes: one % uniformly moves all jobs to the front, and the other throws them back in @@ -86,7 +86,7 @@ % priority 0, and would render this algorithm useless. The default value of % 0.98 is picked such that if a job ran for one scheduler cycle, then didn't % get to run for 7 hours, it would still have priority > 0. 7 hours was picked -% as it was close enought to 8 hours which is the default maximum error backoff +% as it was close enough to 8 hours which is the default maximum error backoff % interval. % % Example calculation: @@ -215,7 +215,7 @@ decay_priorities() -> % is missing we assume it is 0 clear_zero(?PRIORITIES). -% This is the main part of the alrgorithm. In [1] it is described in the +% This is the main part of the algorithm. In [1] it is described in the % "Priority Adjustment" section. % update_priority(#job{} = Job) -> diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl index 3a6edc0b85e..528ff0f3f2c 100644 --- a/src/couch_replicator/src/couch_replicator_worker.erl +++ b/src/couch_replicator/src/couch_replicator_worker.erl @@ -317,7 +317,7 @@ queue_fetch_loop(#fetch_st{} = St) -> ok = gen_server:call(Parent, {batch_doc, Doc}, infinity) end, lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))), - % Invidually upload docs with attachments. + % Individually upload docs with attachments. maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs1)), {ok, Stats} = gen_server:call(Parent, flush, infinity), ok = report_seq_done(Cp, ReportSeq, Stats), @@ -384,7 +384,7 @@ attempt_revs_diff(#fetch_stats{} = St, NowSec) -> % Update fail ratio. Use the basic exponential moving average formula to smooth % over minor bumps in case we encounter a few % attachments and then get back -% to replicationg documents without attachments. +% to replicating documents without attachments. % update_fetch_stats(#fetch_stats{} = St, Successes, Attempts, Decay, NowSec) -> #fetch_stats{ratio = Avg} = St, diff --git a/src/couch_replicator/src/json_stream_parse.erl b/src/couch_replicator/src/json_stream_parse.erl index 8b0c34fa222..09eff8196e9 100644 --- a/src/couch_replicator/src/json_stream_parse.erl +++ b/src/couch_replicator/src/json_stream_parse.erl @@ -53,7 +53,7 @@ events(Data, EventFun) when is_binary(Data) -> events(DataFun, EventFun) -> parse_one(DataFun, EventFun, <<>>). -% converts the JSON directly to the erlang represention of Json +% converts the JSON directly to the erlang representation of Json to_ejson(DF) -> {_DF2, EF, _Rest} = events(DF, fun(Ev) -> collect_events(Ev, []) end), [[EJson]] = make_ejson(EF(get_results), [[]]), @@ -63,7 +63,7 @@ to_ejson(DF) -> % % Return this function from inside an event function right after getting an % object_start event. It then collects the remaining events for that object -% and converts it to the erlang represention of Json. +% and converts it to the erlang representation of Json. % % It then calls your ReturnControl function with the erlang object. Your % return control function then should yield another event function. diff --git a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl index 0b4360378d7..ef93902276d 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl @@ -32,13 +32,13 @@ attachment_too_large_replication_test_() -> should_succeed({_Ctx, {Source, Target}}) -> create_doc_with_attachment(Source, <<"doc">>, 1000), config:set("couchdb", "max_attachment_size", "1000", _Persist = false), - ok = replicate(Source, Target), + {ok, _} = replicate(Source, Target), ?assertEqual(ok, compare(Source, Target)). should_fail({_Ctx, {Source, Target}}) -> create_doc_with_attachment(Source, <<"doc">>, 1000), config:set("couchdb", "max_attachment_size", "999", _Persist = false), - ok = replicate(Source, Target), + {ok, _} = replicate(Source, Target), ?assertError({not_found, <<"doc">>}, compare(Source, Target)). create_doc_with_attachment(DbName, DocId, AttSize) -> diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl index 4988bb41df3..788cd13f9a2 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl @@ -16,31 +16,33 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch_replicator/src/couch_replicator.hrl"). +-define(TIMEOUT, 15). + error_reporting_test_() -> { foreach, fun couch_replicator_test_helper:test_setup/0, fun couch_replicator_test_helper:test_teardown/1, [ - ?TDEF_FE(t_fail_bulk_docs), - ?TDEF_FE(t_fail_changes_reader), - ?TDEF_FE(t_fail_doc_put_4xx_well_formed_json_error), - ?TDEF_FE(t_fail_doc_put_4xx_unexpected_json_error), - ?TDEF_FE(t_fail_doc_put_4xx_invalid_json_error), - ?TDEF_FE(t_skip_doc_put_401_errors), - ?TDEF_FE(t_skip_doc_put_403_errors), - ?TDEF_FE(t_skip_doc_put_413_errors), - ?TDEF_FE(t_skip_doc_put_415_errors), - ?TDEF_FE(t_skip_doc_put_invalid_attachment_name), - ?TDEF_FE(t_fail_revs_diff), - ?TDEF_FE(t_fail_bulk_get, 15), - ?TDEF_FE(t_fail_open_docs_get, 15), - ?TDEF_FE(t_fail_changes_queue), - ?TDEF_FE(t_fail_changes_manager), - ?TDEF_FE(t_fail_changes_reader_proc), - ?TDEF_FE(t_dont_start_duplicate_job), - ?TDEF_FE(t_can_start_multiple_jobs), - ?TDEF_FE(t_stop_duplicate_job) + ?TDEF_FE(t_fail_bulk_docs, ?TIMEOUT), + ?TDEF_FE(t_fail_changes_reader, ?TIMEOUT), + ?TDEF_FE(t_fail_doc_put_4xx_well_formed_json_error, ?TIMEOUT), + ?TDEF_FE(t_fail_doc_put_4xx_unexpected_json_error, ?TIMEOUT), + ?TDEF_FE(t_fail_doc_put_4xx_invalid_json_error, ?TIMEOUT), + ?TDEF_FE(t_skip_doc_put_401_errors, ?TIMEOUT), + ?TDEF_FE(t_skip_doc_put_403_errors, ?TIMEOUT), + ?TDEF_FE(t_skip_doc_put_413_errors, ?TIMEOUT), + ?TDEF_FE(t_skip_doc_put_415_errors, ?TIMEOUT), + ?TDEF_FE(t_skip_doc_put_invalid_attachment_name, ?TIMEOUT), + ?TDEF_FE(t_fail_revs_diff, ?TIMEOUT), + ?TDEF_FE(t_fail_bulk_get, ?TIMEOUT), + ?TDEF_FE(t_fail_open_docs_get, ?TIMEOUT), + ?TDEF_FE(t_fail_changes_queue, ?TIMEOUT), + ?TDEF_FE(t_fail_changes_manager, ?TIMEOUT), + ?TDEF_FE(t_fail_changes_reader_proc, ?TIMEOUT), + ?TDEF_FE(t_dont_start_duplicate_job, ?TIMEOUT), + ?TDEF_FE(t_can_start_multiple_jobs, ?TIMEOUT), + ?TDEF_FE(t_stop_duplicate_job, ?TIMEOUT) ] }. diff --git a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl index e60e3be5d0a..166b36c6ce7 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl @@ -44,7 +44,7 @@ large_atts_test_() -> should_replicate_atts({_Ctx, {Source, Target}}) -> populate_db(Source, ?DOCS_COUNT), - ?assertEqual(ok, replicate(Source, Target)), + ?assertMatch({ok, _}, replicate(Source, Target)), couch_replicator_test_helper:cluster_compare_dbs(Source, Target). populate_db(DbName, DocCount) -> diff --git a/src/couch_replicator/test/eunit/couch_replicator_scheduler_docs_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_scheduler_docs_tests.erl index 7cf79da771d..b3bd7b543e2 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_scheduler_docs_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_scheduler_docs_tests.erl @@ -48,7 +48,6 @@ setup_prefixed_replicator_db_with_update_docs_true() -> {Ctx, {RepDb, Source, Target}}. teardown({Ctx, {RepDb, Source, Target}}) -> - meck:unload(), ok = fabric:delete_db(RepDb, [?ADMIN_CTX]), config:delete("replicator", "update_docs", _Persist = false), couch_replicator_test_helper:test_teardown({Ctx, {Source, Target}}). @@ -76,7 +75,7 @@ scheduler_docs_test_prefixed_db_test_() -> replicator_bdu_test_main_db_test_() -> { setup, - fun setup_prefixed_replicator_db/0, + fun setup_main_replicator_db/0, fun teardown/1, with([ ?TDEF(t_local_docs_can_be_written), diff --git a/src/couch_replicator/test/eunit/couch_replicator_scheduler_job_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_scheduler_job_tests.erl new file mode 100644 index 00000000000..a3f47783685 --- /dev/null +++ b/src/couch_replicator/test/eunit/couch_replicator_scheduler_job_tests.erl @@ -0,0 +1,330 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_replicator_scheduler_job_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-define(CHANGES_READER, couch_replicator_changes_reader). +-define(DOC(Id), #{<<"_id">> => integer_to_binary(Id)}). +-define(DOCS(StartId, StopId), #{ + <<"docs">> => + [ + #{<<"_id">> => integer_to_binary(Id)} + || Id <- lists:seq(StartId, StopId) + ] +}). +-define(JSON, {"Content-Type", "application/json"}). + +setup_replicator_db(Prefix) -> + RepDb = + case Prefix of + <<>> -> <<"_replicator">>; + <<_/binary>> -> <> + end, + Opts = [{q, 1}, {n, 1}, ?ADMIN_CTX], + case fabric:create_db(RepDb, Opts) of + ok -> ok; + {error, file_exists} -> ok + end, + RepDb. + +setup_main_replicator_db() -> + {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(), + RepDb = setup_replicator_db(<<>>), + meck:new(?CHANGES_READER, [passthrough]), + {Ctx, {RepDb, Source, Target}}. + +setup_prefixed_replicator_db() -> + {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(), + RepDb = setup_replicator_db(?tempdb()), + meck:new(?CHANGES_READER, [passthrough]), + {Ctx, {RepDb, Source, Target}}. + +teardown({Ctx, {RepDb, Source, Target}}) -> + ok = fabric:delete_db(RepDb, [?ADMIN_CTX]), + config:delete("replicator", "update_docs", _Persist = false), + couch_replicator_test_helper:test_teardown({Ctx, {Source, Target}}). + +scheduler_job_replicate_test_() -> + { + foreach, + fun setup_main_replicator_db/0, + fun teardown/1, + [ + ?TDEF_FE(t_replicate_without_since_seq), + ?TDEF_FE(t_replicate_with_since_seq_only), + ?TDEF_FE(t_replicate_with_checkpoint_and_since_seq) + ] + }. + +scheduler_job_main_db_test_() -> + { + foreach, + fun setup_main_replicator_db/0, + fun teardown/1, + [ + ?TDEF_FE(t_replicator_without_since_seq, 15), + ?TDEF_FE(t_replicator_with_since_seq_only, 15), + ?TDEF_FE(t_replicator_with_checkpoint_and_since_seq, 25) + ] + }. + +scheduler_job_prefixed_db_test_() -> + { + foreach, + fun setup_prefixed_replicator_db/0, + fun teardown/1, + [ + ?TDEF_FE(t_replicator_without_since_seq, 15), + ?TDEF_FE(t_replicator_with_since_seq_only, 15), + ?TDEF_FE(t_replicator_with_checkpoint_and_since_seq, 25) + ] + }. + +t_replicate_without_since_seq({_Ctx, {_RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + {ok, RepId1} = replicate(Source, Target), + ?assertEqual(1, num_calls(read_changes, ['_', 0, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 3}, all_docs(Target)), + + meck:reset(?CHANGES_READER), + ok = create_doc(Source, ?DOC(4)), + {ok, RepId2} = replicate(Source, Target), + Changes = changes(Source), + Seq = sequence(?DOC(3), Changes), + ?assertEqual(RepId1, RepId2), + ?assertEqual(1, num_calls(read_changes, ['_', Seq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 4}, all_docs(Target)). + +t_replicate_with_since_seq_only({_Ctx, {_RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + Changes = changes(Source), + SinceSeq = sequence(?DOC(2), Changes), + replicate(Source, Target, SinceSeq), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 1}, all_docs(Target)). + +t_replicate_with_checkpoint_and_since_seq({_Ctx, {_RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + Changes = changes(Source), + SinceSeq = sequence(?DOC(2), Changes), + {ok, RepId1} = replicate(Source, Target, SinceSeq), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 1}, all_docs(Target)), + + % Replicate with the checkpoint and ignore `since_seq`. + meck:reset(?CHANGES_READER), + ok = create_doc(Source, ?DOC(4)), + {ok, RepId2} = replicate(Source, Target, SinceSeq), + Seq = sequence(?DOC(3), Changes), + ?assertEqual(RepId1, RepId2), + ?assertEqual(1, num_calls(read_changes, ['_', Seq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 2}, all_docs(Target)), + + % No checkpoint exist, so replicate with the `since_seq`. + meck:reset(?CHANGES_READER), + ok = create_docs(Source, ?DOCS(5, 7)), + Changes1 = changes(Source), + SinceSeq1 = sequence(?DOC(6), Changes1), + {ok, RepId3} = replicate(Source, Target, SinceSeq1), + ?assertNotEqual(RepId2, RepId3), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq1, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 3}, all_docs(Target)). + +t_replicator_without_since_seq({_Ctx, {RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + SourceUrl = couch_replicator_test_helper:cluster_db_url(Source), + TargetUrl = couch_replicator_test_helper:cluster_db_url(Target), + RepDoc = #{<<"source">> => SourceUrl, <<"target">> => TargetUrl}, + {RepDocId, RepId1} = persistent_replicate(RepDb, RepDoc), + ?assertEqual(1, num_calls(read_changes, ['_', 0, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 3}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId)), + + meck:reset(?CHANGES_READER), + ok = create_doc(Source, ?DOC(4)), + {RepDocId2, RepId2} = persistent_replicate(RepDb, RepDoc), + Changes = changes(Source), + Seq = sequence(?DOC(3), Changes), + ?assertEqual(RepId1, RepId2), + ?assertEqual(1, num_calls(read_changes, ['_', Seq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 4}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId2)). + +t_replicator_with_since_seq_only({_Ctx, {RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + Changes = changes(Source), + SinceSeq = sequence(?DOC(2), Changes), + SourceUrl = couch_replicator_test_helper:cluster_db_url(Source), + TargetUrl = couch_replicator_test_helper:cluster_db_url(Target), + RepDoc = #{<<"source">> => SourceUrl, <<"target">> => TargetUrl, <<"since_seq">> => SinceSeq}, + {RepDocId, _} = persistent_replicate(RepDb, RepDoc), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 1}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId)). + +t_replicator_with_checkpoint_and_since_seq({_Ctx, {RepDb, Source, Target}}) -> + ok = create_docs(Source, ?DOCS(1, 3)), + Changes = changes(Source), + SinceSeq = sequence(?DOC(2), Changes), + SourceUrl = couch_replicator_test_helper:cluster_db_url(Source), + TargetUrl = couch_replicator_test_helper:cluster_db_url(Target), + RepDoc = #{<<"source">> => SourceUrl, <<"target">> => TargetUrl, <<"since_seq">> => SinceSeq}, + {RepDocId, RepId1} = persistent_replicate(RepDb, RepDoc), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 1}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId)), + + % Old replication: checkpoint exist, so replicate with the checkpoint. + meck:reset(?CHANGES_READER), + ok = create_doc(Source, ?DOC(4)), + {RepDocId2, RepId2} = persistent_replicate(RepDb, RepDoc), + Seq = sequence(?DOC(3), Changes), + ?assertEqual(RepId1, RepId2), + ?assertEqual(1, num_calls(read_changes, ['_', Seq, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 2}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId2)), + + % New replication: no checkpoint exist, so replicate with the `since_seq`. + meck:reset(?CHANGES_READER), + ok = create_docs(Source, ?DOCS(5, 7)), + Changes1 = changes(Source), + SinceSeq1 = sequence(?DOC(6), Changes1), + RepDoc1 = #{<<"source">> => SourceUrl, <<"target">> => TargetUrl, <<"since_seq">> => SinceSeq1}, + {RepDocId3, RepId3} = persistent_replicate(RepDb, RepDoc1), + ?assertNotEqual(RepId2, RepId3), + ?assertEqual(1, num_calls(read_changes, ['_', SinceSeq1, '_', '_', '_'])), + ?assertMatch(#{<<"total_rows">> := 3}, all_docs(Target)), + ?assertEqual(null, scheduler_docs_id(RepDb, RepDocId3)). + +%%%%%%%%%%%%%%%%%%%% Utility Functions %%%%%%%%%%%%%%%%%%%% +url(UrlPath) -> + binary_to_list(couch_replicator_test_helper:cluster_db_url(UrlPath)). + +create_docs(DbName, Docs) -> + case req(post, url(DbName) ++ "/_bulk_docs", Docs) of + {201, _} -> ok; + Error -> error({failed_to_create_docs, DbName, Error}) + end. + +create_doc(DbName, Doc) -> + case req(post, url(DbName), Doc) of + {201, _} -> ok; + Error -> error({failed_to_create_doc, DbName, Error}) + end. + +all_docs(DbName) -> + {200, Res} = req(get, url(DbName) ++ "/_all_docs"), + ?assert(maps:is_key(<<"offset">>, Res)), + ?assert(maps:is_key(<<"rows">>, Res)), + ?assert(maps:is_key(<<"total_rows">>, Res)), + Res. + +changes(DbName) -> + {200, Res} = req(get, url(DbName) ++ "/_changes"), + ?assert(maps:is_key(<<"last_seq">>, Res)), + ?assert(maps:is_key(<<"pending">>, Res)), + ?assert(maps:is_key(<<"results">>, Res)), + Res. + +sequence(Doc, Changes) -> + #{<<"_id">> := DocId} = Doc, + #{<<"results">> := Results} = Changes, + case lists:search(fun(M) -> maps:get(<<"id">>, M) == DocId end, Results) of + {value, #{<<"seq">> := Seq}} -> Seq; + false -> not_found + end. + +replicate(RepObject) -> + couch_replicator_test_helper:replicate(RepObject). + +replicate(Source, Target) -> + replicate(#{ + <<"source">> => ?l2b(url(Source)), + <<"target">> => ?l2b(url(Target)) + }). + +replicate(Source, Target, SinceSeq) -> + replicate(#{ + <<"source">> => ?l2b(url(Source)), + <<"target">> => ?l2b(url(Target)), + <<"since_seq">> => SinceSeq + }). + +persistent_replicate(RepDb, RepDoc) -> + RepDocId = ?docid(), + rep_toggle(stop), + RepDocUrl = rep_doc_url(RepDb, RepDocId), + {201, _} = req(put, RepDocUrl, RepDoc), + RepId = scheduler_docs_id(RepDb, RepDocId), + rep_toggle(start), + ok = test_util:wait( + fun() -> + case req(get, RepDocUrl) of + {200, #{<<"_replication_state">> := <<"completed">>}} -> ok; + {_, #{}} -> wait + end + end, + 7000, + 1000 + ), + {RepDocId, RepId}. + +rep_toggle(start) -> + config:set("replicator", "max_jobs", "500", false); +rep_toggle(stop) -> + config:set("replicator", "max_jobs", "0", false). + +rep_doc_url(RepDb, DocId) when is_binary(RepDb) -> + rep_doc_url(binary_to_list(RepDb), DocId); +rep_doc_url(RepDb, DocId) when is_binary(DocId) -> + rep_doc_url(RepDb, binary_to_list(DocId)); +rep_doc_url(RepDb, DocId) when is_list(RepDb), is_list(DocId) -> + UrlQuotedRepDb = mochiweb_util:quote_plus(RepDb), + url(UrlQuotedRepDb ++ "/" ++ DocId). + +scheduler_docs_id(RepDb, RepDocId) -> + RepDocIdBin = ?l2b(RepDocId), + SchedulerDocsUrl = + case RepDb of + <<"_replicator">> -> url(<<"/_scheduler/docs">>); + <<_/binary>> -> url(<<"/_scheduler/docs/", RepDb/binary>>) + end, + Docs = test_util:wait( + fun() -> + case req(get, SchedulerDocsUrl) of + {200, #{<<"docs">> := [_ | _] = Docs}} -> Docs; + {200, #{<<"docs">> := []}} -> wait + end + end, + 7000, + 1000 + ), + [RepId] = [Id || #{<<"doc_id">> := DocId, <<"id">> := Id} <- Docs, DocId =:= RepDocIdBin], + RepId. + +req(Method, Url) -> + Headers = [?JSON], + {ok, Code, _, Res} = test_request:request(Method, Url, Headers), + {Code, jiffy:decode(Res, [return_maps])}. + +req(Method, Url, #{} = Body) -> + req(Method, Url, jiffy:encode(Body)); +req(Method, Url, Body) -> + Headers = [?JSON], + {ok, Code, _, Res} = test_request:request(Method, Url, Headers, Body), + {Code, jiffy:decode(Res, [return_maps])}. + +num_calls(Fun, Args) -> + meck:num_calls(?CHANGES_READER, Fun, Args). diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl index 5f2cfa25f21..39f803cf5b6 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl @@ -186,7 +186,8 @@ replicate({[_ | _]} = RepObject) -> {'DOWN', MonRef, process, Pid, _} -> ok end, - ok = couch_replicator_scheduler:remove_job(Rep#rep.id). + ok = couch_replicator_scheduler:remove_job(Rep#rep.id), + {ok, Rep#rep.id}. setup_db() -> DbName = ?tempdb(), diff --git a/src/couch_replicator/test/eunit/fixtures/logo.png b/src/couch_replicator/test/eunit/fixtures/logo.png new file mode 100644 index 00000000000..d21ac025b52 Binary files /dev/null and b/src/couch_replicator/test/eunit/fixtures/logo.png differ diff --git a/src/couch_scanner/src/couch_scanner_plugin.erl b/src/couch_scanner/src/couch_scanner_plugin.erl index 301f5e6ffb5..b31ed949ff3 100644 --- a/src/couch_scanner/src/couch_scanner_plugin.erl +++ b/src/couch_scanner/src/couch_scanner_plugin.erl @@ -41,20 +41,27 @@ % scan starts from the beginning (first db, first shard, ...), and resume/2 is % called when the scanning hasn't finished and has to continue. % -% If start/2 or resume/2 returns `reset` then the checkpoint will be reset and -% the plugin will be restarted. This may be useful in cases when the plugin -% detects configuration changes since last scanning session had already -% started, or when the plugin module was updated and the checkpoint version is -% stale. -% % The checkpoint/1 callback is periodically called to checkpoint the scanning % progress. start/2 and resume/2 function will be called with the last saved % checkpoint map value. % +% If start/2, resume/2 or checkpoint/1 returns `reset` then the checkpoint will +% be reset and the plugin will be restarted. This may be useful in cases when +% the plugin detects configuration changes since last scanning session had +% already started, or when the plugin module was updated and the checkpoint +% version is stale. +% +% To stop or pause execution of the plugin, start/2 and resume/2 may return +% `skip` and checkpoint/1 can return `{stop, State}`. Skipping on start or +% resume counts as a "run" at that time and the plugin is scheduled to run next +% time according to it schedule. A `{stop, State}` return from a checkpoint +% will persist the `State` in the checkpoint and next time the plugin will +% resume from that checkpoint. +% % The complete/1 callback is called when the scan has finished. The complete -% callback should return final checkpoint map object. The last checkpoint will -% be written and then it will be passed to the start/2 callback if the plugin -% runs again. +% callback should return a final checkpoint map object. The last checkpoint +% will be written, and then it will be passed to the start/2 callback if the +% plugin runs again. % % As the cluster dbs, shards, ddocs and individual docs are discovered during % scanning, the appropriate callbacks will be called. Most callbacks, besides @@ -100,7 +107,7 @@ % Optional -callback checkpoint(St :: term()) -> - {ok, EJson :: #{}}. + {ok | stop, EJson :: #{}} | reset. -callback db(St :: term(), DbName :: binary()) -> {ok | skip | stop, St1 :: term()}. @@ -444,13 +451,14 @@ rate_limit(#st{rlimiter = RLimiter} = St, Type) -> checkpoint(#st{} = St) -> #st{ id = Id, + mod = Mod, callbacks = Cbks, pst = PSt, cursor = Cursor, start_sec = StartSec, scan_id = SId } = St, - EJsonPSt = checkpoint_callback(Cbks, PSt), + {Go, EJsonPSt} = checkpoint_callback(Cbks, PSt), EJson = #{ <<"cursor">> => Cursor, <<"pst">> => EJsonPSt, @@ -459,7 +467,16 @@ checkpoint(#st{} = St) -> <<"start_sec">> => StartSec }, ok = couch_scanner_checkpoint:write(Id, EJson), - St#st{checkpoint_sec = tsec()}. + case Go of + ok -> + St#st{checkpoint_sec = tsec()}; + stop -> + % Plugin wants to stop. Exit and let it reschedule again for + % whatever scheduling setup it's configured with. Next time + % it runs, it will resume from the checkpoint it just performed. + ReschedTSec = schedule_time(Mod, StartSec, tsec()), + exit_resched(ReschedTSec) + end. finalize(#st{} = St) -> #st{ @@ -551,7 +568,7 @@ shards_callback(#st{pst = PSt, callbacks = Cbks} = St, Shards) -> start_checkpoint(Id, #{} = Cbks, StartSec, ScanId, Cur, PSt) when is_binary(Id), is_binary(ScanId), is_integer(StartSec) -> - EJsonPSt = checkpoint_callback(Cbks, PSt), + {ok, EJsonPSt} = checkpoint_callback(Cbks, PSt), EJson = #{ <<"cursor">> => Cur, <<"pst">> => EJsonPSt, @@ -564,7 +581,8 @@ start_checkpoint(Id, #{} = Cbks, StartSec, ScanId, Cur, PSt) when checkpoint_callback(#{} = Cbks, PSt) -> #{checkpoint := CheckpointCbk} = Cbks, case CheckpointCbk(PSt) of - {ok, #{} = EJsonPSt} -> couch_scanner_util:ejson_map(EJsonPSt); + {ok, #{} = EJsonPSt} -> {ok, couch_scanner_util:ejson_map(EJsonPSt)}; + {stop, #{} = EJsonPSt} -> {stop, couch_scanner_util:ejson_map(EJsonPSt)}; reset -> exit_resched(reset) end. diff --git a/src/couch_scanner/src/couch_scanner_server.erl b/src/couch_scanner/src/couch_scanner_server.erl index 2ec1ac54084..ecb7e73f0dd 100644 --- a/src/couch_scanner/src/couch_scanner_server.erl +++ b/src/couch_scanner/src/couch_scanner_server.erl @@ -177,7 +177,7 @@ stop_in_maintenance(#st{pids = Pids} = St) -> end. start_stop(#st{stopped = Stopped} = St) -> - case in_maintenance() orelse Stopped of + case in_maintenance() orelse Stopped orelse upgrade_in_progress() of true -> stop_in_maintenance(St); false -> start_stop_cfg(St) end. @@ -279,6 +279,9 @@ penalize(Now, #sched{error_count = ErrorCount} = Sched) -> in_maintenance() -> "false" /= config:get("couchdb", "maintenance_mode", "false"). +upgrade_in_progress() -> + config:get_boolean("couchdb", "upgrade_in_progress", false). + tsec() -> erlang:system_time(second). diff --git a/src/couch_scanner/test/eunit/couch_scanner_test.erl b/src/couch_scanner/test/eunit/couch_scanner_test.erl index 5d6a22f3871..9cce06f3824 100644 --- a/src/couch_scanner/test/eunit/couch_scanner_test.erl +++ b/src/couch_scanner/test/eunit/couch_scanner_test.erl @@ -23,6 +23,9 @@ couch_scanner_test_() -> [ ?TDEF_FE(t_top_level_api), ?TDEF_FE(t_start_stop), + ?TDEF_FE(t_start_stop_mm_mode, 10), + ?TDEF_FE(t_start_stop_upgrade_in_progress, 10), + ?TDEF_FE(t_stop_auto_purge_on_dead_nodes, 10), ?TDEF_FE(t_run_through_all_callbacks_basic, 10), ?TDEF_FE(t_find_reporting_works, 10), ?TDEF_FE(t_ddoc_features_works, 20), @@ -46,13 +49,17 @@ couch_scanner_test_() -> -define(FIND_PLUGIN, couch_scanner_plugin_find). -define(FEATURES_PLUGIN, couch_scanner_plugin_ddoc_features). -define(CONFLICTS_PLUGIN, couch_scanner_plugin_conflict_finder). +-define(AUTO_PURGE_PLUGIN, couch_auto_purge_plugin). setup() -> {module, _} = code:ensure_loaded(?FIND_PLUGIN), + {module, _} = code:ensure_loaded(?AUTO_PURGE_PLUGIN), meck:new(?FIND_PLUGIN, [passthrough]), + meck:new(?AUTO_PURGE_PLUGIN, [passthrough]), meck:new(fabric, [passthrough]), meck:new(couch_scanner_server, [passthrough]), meck:new(couch_scanner_util, [passthrough]), + meck:new(mem3, [passthrough]), Ctx = test_util:start_couch([fabric, couch_scanner]), % Run with the smallest batch size to exercise the batched % ddoc iteration @@ -100,11 +107,13 @@ setup() -> {Ctx, {DbName1, DbName2, DbName3}}. teardown({Ctx, {DbName1, DbName2, DbName3}}) -> - config:delete("couch_scanner", "maintenance_mode", false), + config:delete("couchdb", "maintenance_mode", false), + config:delete("couchdb", "upgrade_in_progress", false), config_delete_section("couch_scanner"), config_delete_section("couch_scanner_plugins"), config_delete_section(atom_to_list(?FEATURES_PLUGIN)), config_delete_section(atom_to_list(?FIND_PLUGIN)), + config_delete_section(atom_to_list(?AUTO_PURGE_PLUGIN)), config_delete_section(atom_to_list(?CONFLICTS_PLUGIN)), lists:foreach( fun(Subsection) -> @@ -139,6 +148,49 @@ t_start_stop(_) -> ?assertEqual(ok, couch_scanner_server:resume()), ?assertMatch(#{stopped := false}, couch_scanner:status()). +t_start_stop_mm_mode(_) -> + ?assertEqual(ok, couch_scanner:stop()), + Plugin = atom_to_list(?FIND_PLUGIN), + config:set("couch_scanner_plugins", Plugin, "true", false), + meck:expect(?FIND_PLUGIN, shards, fun(_, _) -> timer:sleep(10000) end), + config:set("couchdb", "maintenance_mode", "true", true), + ?assertEqual(ok, couch_scanner:resume()), + #{pids := Pids1, stopped := false} = couch_scanner:status(), + ?assertEqual(#{}, Pids1), + config:set("couchdb", "maintenance_mode", "false", true), + ?assertEqual(ok, couch_scanner:stop()), + ?assertEqual(ok, couch_scanner:resume()), + #{pids := Pids2, stopped := false} = couch_scanner:status(), + ?assertMatch(#{<<"couch_scanner_plugin_find">> := Pid} when is_pid(Pid), Pids2), + ?assertEqual(ok, couch_scanner:stop()). + +t_start_stop_upgrade_in_progress(_) -> + ?assertEqual(ok, couch_scanner:stop()), + Plugin = atom_to_list(?FIND_PLUGIN), + config:set("couch_scanner_plugins", Plugin, "true", false), + meck:expect(?FIND_PLUGIN, shards, fun(_, _) -> timer:sleep(10000) end), + config:set("couchdb", "upgrade_in_progress", "true", true), + ?assertEqual(ok, couch_scanner:resume()), + #{pids := Pids1, stopped := false} = couch_scanner:status(), + ?assertEqual(#{}, Pids1), + config:set("couchdb", "upgrade_in_progress", "false", true), + ?assertEqual(ok, couch_scanner:stop()), + ?assertEqual(ok, couch_scanner:resume()), + #{pids := Pids2, stopped := false} = couch_scanner:status(), + ?assertMatch(#{<<"couch_scanner_plugin_find">> := Pid} when is_pid(Pid), Pids2), + ?assertEqual(ok, couch_scanner:stop()). + +t_stop_auto_purge_on_dead_nodes(_) -> + meck:reset(couch_scanner_server), + meck:reset(couch_scanner_util), + meck:expect(mem3, nodes, fun() -> ['potato@127.0.0.1'] end), + Plugin = atom_to_list(?AUTO_PURGE_PLUGIN), + config:set("couch_scanner_plugins", Plugin, "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(?AUTO_PURGE_PLUGIN, start, 2)), + ?assertEqual(0, num_calls(?AUTO_PURGE_PLUGIN, complete, 1)), + ?assertEqual(1, log_calls(?AUTO_PURGE_PLUGIN, info)). + t_run_through_all_callbacks_basic({_, {DbName1, DbName2, _}}) -> % Run the "find" plugin without any regexes meck:reset(couch_scanner_server), @@ -351,10 +403,16 @@ add_docs(DbName, Docs) -> {ok, []} = fabric:update_docs(DbName, Docs, [?REPLICATED_CHANGES, ?ADMIN_CTX]). num_calls(Fun, Args) -> - meck:num_calls(?FIND_PLUGIN, Fun, Args). + num_calls(?FIND_PLUGIN, Fun, Args). + +num_calls(Mod, Fun, Args) -> + meck:num_calls(Mod, Fun, Args). log_calls(Level) -> - meck:num_calls(couch_scanner_util, log, [Level, ?FIND_PLUGIN, '_', '_', '_']). + log_calls(?FIND_PLUGIN, Level). + +log_calls(Mod, Level) -> + meck:num_calls(couch_scanner_util, log, [Level, Mod, '_', '_', '_']). wait_exit(MSec) -> meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'], MSec). diff --git a/src/couch_stats/src/couch_stats.erl b/src/couch_stats/src/couch_stats.erl index 29a4024491f..c761614c83c 100644 --- a/src/couch_stats/src/couch_stats.erl +++ b/src/couch_stats/src/couch_stats.erl @@ -14,7 +14,6 @@ -export([ fetch/0, - reload/0, sample/1, increment_counter/1, increment_counter/2, @@ -33,9 +32,6 @@ fetch() -> % Last -1 is because the interval ends are inclusive couch_stats_util:fetch(stats(), StartSec, Seconds). -reload() -> - couch_stats_server:reload(). - -spec sample(any()) -> stat(). sample(Name) -> Seconds = couch_stats_util:histogram_interval_sec(), @@ -112,7 +108,6 @@ couch_stats_test_() -> [ ?TDEF_FE(t_fetch_metrics), ?TDEF_FE(t_sample_metrics), - ?TDEF_FE(t_reload), ?TDEF_FE(t_increment_counter), ?TDEF_FE(t_decrement_counter), ?TDEF_FE(t_update_gauge), @@ -144,10 +139,6 @@ t_sample_metrics(_) -> ?assertEqual(0, sample([couch_replicator, jobs, total])). -t_reload(_) -> - % This is tested in detail in couch_stats_server. - ?assertEqual(ok, reload()). - t_increment_counter(_) -> [increment_counter([fsync, count]) || _ <- lists:seq(1, 1000)], ?assert(sample([fsync, count]) > 1000). @@ -190,8 +181,6 @@ t_access_invalid_metrics(_) -> ?assertEqual({error, invalid_metric}, decrement_counter([fsync, time], 100)), ?assertEqual({error, invalid_metric}, update_gauge([fsync, count], 100)), ?assertEqual({error, invalid_metric}, update_histogram([fsync, count], 100)), - ?assertThrow({invalid_metric, _}, update_histogram([fsync, count], Fun)), - InvalidMetrics = #{[bad] => {invalid, <<"desc">>}}, - ?assertThrow({unknown_metric, _}, couch_stats_util:create_metrics(InvalidMetrics)). + ?assertThrow({invalid_metric, _}, update_histogram([fsync, count], Fun)). -endif. diff --git a/src/couch_stats/src/couch_stats_math.erl b/src/couch_stats/src/couch_stats_math.erl index dfffc9a1651..f4162fbf646 100644 --- a/src/couch_stats/src/couch_stats_math.erl +++ b/src/couch_stats/src/couch_stats_math.erl @@ -241,7 +241,23 @@ n0_stats() -> -include_lib("couch/include/couch_eunit.hrl"). -basic_test() -> +-define(TIMEOUT, 10). + +couch_stats_test_() -> + { + setup, + fun() -> ok end, + fun(_) -> ok end, + with([ + ?TDEF(t_basic, ?TIMEOUT), + ?TDEF(t_min_extreme, ?TIMEOUT), + ?TDEF(t_max_extreme, ?TIMEOUT), + ?TDEF(t_normal_dist, ?TIMEOUT), + ?TDEF(t_uniform_dist, ?TIMEOUT) + ]) + }. + +t_basic(_) -> H = couch_stats_histogram:new(), Vals = [0.05, 0.9, 0.7, 0.7, 10.1, 11, 100.5, 0.10, 13.5], [couch_stats_histogram:update(H, V) || V <- Vals], @@ -270,7 +286,7 @@ basic_test() -> ?assert(flim(103, prop(99, Percentiles))), ?assert(flim(104, prop(999, Percentiles))). -min_extreme_test() -> +t_min_extreme(_) -> % All the values in the smallest bin H = couch_stats_histogram:new(), N = 1000000, @@ -295,7 +311,7 @@ min_extreme_test() -> ?assert(flim(0, prop(99, Percentiles))), ?assert(flim(0, prop(999, Percentiles))). -max_extreme_test() -> +t_max_extreme(_) -> % All the values are in the largest bin H = couch_stats_histogram:new(), N = 1000000, @@ -327,7 +343,7 @@ max_extreme_test() -> ?assert(flim(4191682, prop(99, Percentiles))), ?assert(flim(4194041, prop(999, Percentiles))). -normal_dist_test() -> +t_normal_dist(_) -> H = couch_stats_histogram:new(), rand:seed(default, {1, 2, 3}), N = 1000000, @@ -359,7 +375,7 @@ normal_dist_test() -> ?assert(flim(74, prop(99, Percentiles))), ?assert(flim(82, prop(999, Percentiles))). -uniform_dist_test() -> +t_uniform_dist(_) -> H = couch_stats_histogram:new(), rand:seed(default, {1, 2, 3}), N = 1000000, diff --git a/src/couch_stats/src/couch_stats_server.erl b/src/couch_stats/src/couch_stats_server.erl index d58a8f06156..640c1fb2372 100644 --- a/src/couch_stats/src/couch_stats_server.erl +++ b/src/couch_stats/src/couch_stats_server.erl @@ -10,21 +10,14 @@ % License for the specific language governing permissions and limitations under % the License. -% couch_stats_server is in charge of: -% - Initial metric loading from application stats descriptions. -% - Recycling(resetting to 0) stale histogram counters. -% - Checking and reloading if stats descriptions change. -% - Checking and reloading if histogram interval config value changes. +% couch_stats_server is in charge of initially loading stats definition into a +% persistent term then recycling(resetting to 0) stale histogram counters. % -module(couch_stats_server). -behaviour(gen_server). --export([ - reload/0 -]). - -export([ start_link/0, init/1, @@ -33,45 +26,53 @@ handle_info/2 ]). --define(RELOAD_INTERVAL_SEC, 600). +% config_listener +-export([ + handle_config_change/5, + handle_config_terminate/3 +]). -record(st, { - hist_interval, histograms, - clean_tref, - reload_tref + clean_tref }). -reload() -> - gen_server:call(?MODULE, reload). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> + ok = config:listen_for_changes(?MODULE, self()), St = #st{ - hist_interval = config:get("stats", "interval"), clean_tref = erlang:send_after(clean_msec(), self(), clean), - reload_tref = erlang:send_after(reload_msec(), self(), reload) + histograms = couch_stats_util:histograms(couch_stats_util:load()) }, - {_, Stats} = try_reload(St), - {ok, St#st{histograms = couch_stats_util:histograms(Stats)}}. + {ok, St}. -handle_call(reload, _From, #st{} = St) -> - {reply, ok, do_reload(St)}; handle_call(Msg, _From, #st{} = St) -> {stop, {unknown_call, Msg}, unknown_call, St}. handle_cast(Msg, #st{} = St) -> {stop, {unknown_cast, Msg}, St}. -handle_info(reload, #st{} = St) -> - {noreply, do_reload(St)}; handle_info(clean, #st{} = St) -> {noreply, do_clean(St)}; +handle_info(restart_config_listener, #st{} = St) -> + ok = config:listen_for_changes(?MODULE, self()), + {noreply, St}; handle_info(Msg, #st{} = St) -> {stop, {unknown_info, Msg}, St}. +handle_config_change("stats", "interval", _, _, Pid) -> + Pid ! clean, + {ok, Pid}; +handle_config_change(_, _, _, _, Pid) -> + {ok, Pid}. + +handle_config_terminate(_, stop, _) -> + ok; +handle_config_terminate(_Server, _Reason, Pid) -> + erlang:send_after(1000, Pid, restart_config_listener). + do_clean(#st{} = St) -> timer:cancel(St#st.clean_tref), HistTRef = erlang:send_after(clean_msec(), self(), clean), @@ -97,50 +98,6 @@ do_clean(#st{} = St) -> ), St#st{clean_tref = HistTRef}. -do_reload(#st{} = St) -> - timer:cancel(St#st.reload_tref), - RTRef = erlang:send_after(reload_msec(), self(), reload), - case try_reload(St) of - {true, NewStats} -> - timer:cancel(St#st.clean_tref), - Histograms = couch_stats_util:histograms(NewStats), - HTRef = erlang:send_after(clean_msec(), self(), clean), - St#st{ - histograms = Histograms, - clean_tref = HTRef, - reload_tref = RTRef, - hist_interval = config:get("stats", "interval") - }; - {false, _} -> - St#st{reload_tref = RTRef} - end. - -try_reload(#st{} = St) -> - NewDefs = couch_stats_util:load_metrics_for_applications(), - Stats = couch_stats_util:stats(), - MetricsChanged = couch_stats_util:metrics_changed(Stats, NewDefs), - IntervalChanged = interval_changed(St), - case MetricsChanged orelse IntervalChanged of - true -> - couch_stats_util:reset_histogram_interval_sec(), - NewStats = couch_stats_util:create_metrics(NewDefs), - couch_stats_util:replace_stats(NewStats), - {true, NewStats}; - false -> - {false, Stats} - end. - -interval_changed(#st{hist_interval = OldInterval}) -> - case config:get("stats", "interval") of - Interval when OldInterval =:= Interval -> - false; - _ -> - true - end. - -reload_msec() -> - 1000 * ?RELOAD_INTERVAL_SEC. - clean_msec() -> % We want to wake up more often than our interval so we decide to wake % about twice as often. If the interval is 10 seconds, we'd wake up every 5 @@ -159,8 +116,6 @@ couch_stats_server_test_() -> fun teardown/1, [ ?TDEF_FE(t_server_starts), - ?TDEF_FE(t_reload_with_no_changes_works), - ?TDEF_FE(t_reload_with_changes_works), ?TDEF_FE(t_cleaning_works, 10), ?TDEF_FE(t_invalid_call), ?TDEF_FE(t_invalid_cast), @@ -178,50 +133,13 @@ teardown(Ctx) -> t_server_starts(_) -> ?assert(is_process_alive(whereis(?MODULE))). -t_reload_with_no_changes_works(_) -> - Pid = whereis(?MODULE), - ?assert(is_process_alive(Pid)), - ?assertEqual(ok, reload()), - ?assertEqual(Pid, whereis(?MODULE)), - ?assert(is_process_alive(Pid)), - % Let's reload a few more hundred times - lists:foreach( - fun(_) -> - ?assertEqual(ok, reload()), - ?assertEqual(Pid, whereis(?MODULE)), - ?assert(is_process_alive(Pid)) - end, - lists:seq(1, 100) - ). - -t_reload_with_changes_works(_) -> - Pid = whereis(?MODULE), - ?assert(is_process_alive(Pid)), - #st{hist_interval = Interval0} = sys:get_state(Pid), - ?assertEqual(undefined, Interval0), - - config:set("stats", "interval", "7", false), - ?assertEqual(ok, reload()), - ?assertEqual(Pid, whereis(?MODULE)), - ?assert(is_process_alive(Pid)), - #st{hist_interval = Interval1} = sys:get_state(Pid), - ?assertEqual("7", Interval1), - - #st{histograms = Hists} = sys:get_state(Pid), - [{_Key, {histogram, HCtx1, _Desc}} | _] = maps:to_list(Hists), - % Histogram window size should now be shorter - % 7 (active time window) + 7 (stale) + 5 + 5 for buffers = 24. - ?assertEqual(24, tuple_size(HCtx1)). - t_cleaning_works(_) -> config:set("stats", "interval", "1", false), sys:log(?MODULE, {true, 100}), - ok = reload(), timer:sleep(2000), {ok, Events} = sys:log(?MODULE, get), ok = sys:log(?MODULE, false), config:set("stats", "interval", "10", false), - ok = reload(), % Events looks like: [{in, Msg} | {noreply, ...} | {out, ..}, ...] CleanEvents = [clean || {in, clean} <- Events], ?assert(length(CleanEvents) >= 3). diff --git a/src/couch_stats/src/couch_stats_util.erl b/src/couch_stats/src/couch_stats_util.erl index 5cbd54b1d2f..a63d3802330 100644 --- a/src/couch_stats/src/couch_stats_util.erl +++ b/src/couch_stats/src/couch_stats_util.erl @@ -14,9 +14,7 @@ -export([ % Load metrics from apps - create_metrics/1, - load_metrics_for_applications/0, - metrics_changed/2, + load/0, % Get various metric types get_counter/2, @@ -26,10 +24,8 @@ % Get histogram interval config settings histogram_interval_sec/0, histogram_safety_buffer_size_sec/0, - reset_histogram_interval_sec/0, - % Manage the main stats (metrics) persistent term map - replace_stats/1, + % Get the stats persistent term map stats/0, histograms/1, @@ -38,6 +34,8 @@ sample/4 ]). +-include_lib("stdlib/include/assert.hrl"). + -define(DEFAULT_INTERVAL_SEC, 10). % Histogram types @@ -51,41 +49,77 @@ % Persistent term keys -define(STATS_KEY, {?MODULE, stats}). --define(HIST_TIME_INTERVAL_KEY, {?MODULE, hist_time_interval}). + +% Don't waste time looking for stats definition in some built-in and dependency +% apps. This doesn't have to be an exhaustive list, it's just to avoid doing +% extra work. +% +-define(SKIP_APPS, [ + asn1, + b64url, + compiler, + cowlib, + crypto, + gun, + ibrowse, + inets, + jiffy, + kernel, + meck, + mochiweb, + os_mon, + public_key, + rebar, + rebar3, + recon, + runtime_tools, + sasl, + snappy, + ssl, + stdlib, + syntax_tools, + xmerl +]). + +load() -> + Definitions = load_metrics_for_applications(), + Stats = create_metrics(Definitions), + persistent_term:put(?STATS_KEY, Stats), + Stats. load_metrics_for_applications() -> Apps = [element(1, A) || A <- application:loaded_applications()], - lists:foldl(fun load_metrics_for_application_fold/2, #{}, Apps). + Apps1 = [A || A <- Apps, not lists:member(A, ?SKIP_APPS)], + lists:foldl(fun load_metrics_for_application_fold/2, #{}, Apps1). load_metrics_for_application_fold(AppName, #{} = Acc) -> - case code:priv_dir(AppName) of - {error, _Error} -> + % For an existing application we should always be able to compute its + % priv_dir path, even though the directory itself may not exist or may not + % be accessible. + Dir = code:priv_dir(AppName), + ?assert(is_list(Dir), "Could not get application priv_dir " ++ atom_to_list(AppName)), + Path = filename:join(Dir, "stats_descriptions.cfg"), + % Expect some apps not to have stats descriptions and priv_dir paths to not even exist + case file:consult(Path) of + {ok, Descriptions} -> + DescMap = maps:map( + fun(_, TypeDesc) -> + Type = proplists:get_value(type, TypeDesc, counter), + Desc = proplists:get_value(desc, TypeDesc, <<>>), + {Type, Desc} + end, + maps:from_list(Descriptions) + ), + maps:merge(Acc, DescMap); + {error, enoent} -> Acc; - Dir -> - case file:consult(Dir ++ "/stats_descriptions.cfg") of - {ok, Descriptions} -> - DescMap = maps:map( - fun(_, TypeDesc) -> - Type = proplists:get_value(type, TypeDesc, counter), - Desc = proplists:get_value(desc, TypeDesc, <<>>), - {Type, Desc} - end, - maps:from_list(Descriptions) - ), - maps:merge(Acc, DescMap); - {error, _Error} -> - Acc - end + {error, enotdir} -> + Acc; + {error, Error} -> + % Bail if we can't load stats for any other reason + error({couch_stats_load_error, Path, Error}) end. -metrics_changed(#{} = Map1, #{} = Map2) when map_size(Map1) =/= map_size(Map2) -> - % If their sizes are differently they are obvioulsy not the same - true; -metrics_changed(#{} = Map1, #{} = Map2) when map_size(Map1) =:= map_size(Map2) -> - % If their intersection size is not the same as their individual size - % they are also not the same - map_size(maps:intersect(Map1, Map2)) =/= map_size(Map1). - get_counter(Name, #{} = Stats) -> get_metric(Name, ?CNTR, Stats). @@ -108,17 +142,7 @@ get_metric(Name, Type, Stats) when is_atom(Type), is_map(Stats) -> end. histogram_interval_sec() -> - case persistent_term:get(?HIST_TIME_INTERVAL_KEY, not_cached) of - not_cached -> - Time = config:get_integer("stats", "interval", ?DEFAULT_INTERVAL_SEC), - persistent_term:put(?HIST_TIME_INTERVAL_KEY, Time), - Time; - Val when is_integer(Val) -> - Val - end. - -reset_histogram_interval_sec() -> - persistent_term:erase(?HIST_TIME_INTERVAL_KEY). + config:get_integer("stats", "interval", ?DEFAULT_INTERVAL_SEC). histogram_safety_buffer_size_sec() -> ?HIST_WRAP_BUFFER_SIZE_SEC. @@ -128,9 +152,6 @@ histogram_total_size_sec() -> % periodically clear. histogram_interval_sec() * 2 + ?HIST_WRAP_BUFFER_SIZE_SEC * 2. -replace_stats(#{} = Stats) -> - persistent_term:put(?STATS_KEY, Stats). - stats() -> persistent_term:get(?STATS_KEY, #{}). diff --git a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl index 5e2efbffacd..7a860b14567 100644 --- a/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl +++ b/src/ddoc_cache/test/eunit/ddoc_cache_tutil.erl @@ -32,7 +32,7 @@ start_couch() -> start_couch(Options) -> WriteDDocs = couch_util:get_value(write_ddocs, Options, true), purge_modules(), - Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]), + Ctx = test_util:start_couch([chttpd, ddoc_cache]), TmpDb = ?tempdb(), ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]), if diff --git a/src/docs/images/time_seq.png b/src/docs/images/time_seq.png new file mode 100644 index 00000000000..b0023c7ffb4 Binary files /dev/null and b/src/docs/images/time_seq.png differ diff --git a/src/docs/images/time_seq_zoom.png b/src/docs/images/time_seq_zoom.png new file mode 100644 index 00000000000..0c175cd9ace Binary files /dev/null and b/src/docs/images/time_seq_zoom.png differ diff --git a/src/docs/src/api/database/changes.rst b/src/docs/src/api/database/changes.rst index 812e3b5db98..337688fa097 100644 --- a/src/docs/src/api/database/changes.rst +++ b/src/docs/src/api/database/changes.rst @@ -99,10 +99,11 @@ - A valid update sequence, for example, from a ``_changes`` feed response. - ``now`` - ``0`` - - A timestmap string matching the ``YYYY-MM-DDTHH:MM:SSZ`` + - A timestamp string matching the ``YYYY-MM-DDTHH:MM:SSZ`` format. The results returned will depend on the time-sequence intervals recorded by the time-seq data structure. To inspect or reset - it use the :ref:`_time_seq ` endpoint. + it use the :ref:`_time_seq ` endpoint. For additional + details see the :ref:`changes/timeseq` section below. :query string style: Specifies how many revisions are returned in the changes array. The default, ``main_only``, will only return the current "winning" revision; ``all_docs`` will return all leaf revisions @@ -833,3 +834,177 @@ amount of duplicated code. } ] } + +.. _changes/timeseq: + +Time-based changes feeds +======================== + +.. versionadded:: 3.6 + +Starting with version 3.6 :ref:`api/db/changes` API can emit rows starting from +a point in time. To use this feature set the ``since`` parameter value to a +timestamp in the ``YYYY-MM-DDTHH:MM:SSZ`` format. That's a standard format in +both the ISO 8601 and RFC 3339 standards. + +**Request**: + +.. code-block:: http + + GET db/_changes?since=2026-01-09T19:01:02Z HTTP/1.1 + Accept: application/json + Host: localhost:5984 + +**Response**: + +.. code-block:: http + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + + { + "last_seq": "112-g1AAAABLeJzLYWBgYMxgTmHgz8tPSTV0MDQy1zMAQsMcoARTIkMeC8N_IMjKYE4syAUKsZumWJgZGVhgasgCAKx6Erk", + "pending": 0, + "results": [ + { + "changes": [{"rev": "1-967a00dff5e02add41819138abb3284d"}], + "id": "019ba3f4f9b77e06900cf8b631821a3a", + "seq": "111-g1AAAABLeJzLYWBgYMxgTmHgz8tPSTV0MDQy1zMAQsMcoARTIkMeC8N_IMjKYE7MzwUKsZumWJgZGVhgasgCAKxaErg" + }, + { + "changes": [{"rev": "1-967a00dff5e02add41819138abb3284d"}], + "id": "019ba3f51e077f0c82ca136136de2d7f", + "seq": "112-g1AAAABLeJzLYWBgYMxgTmHgz8tPSTV0MDQy1zMAQsMcoARTIkMeC8N_IMjKYE4syAUKsZumWJgZGVhgasgCAKx6Erk" + } + ] + } + +Implementation Details +---------------------- + +CouchDB automatically maps time intervals to db update sequences as documents +get updated. This is implemented with a histogram data structure which looks +like ``[{t3, seq3}, {t2, seq2}, {t1, seq1}, ...]``. + +In order to avoid any performance impact, and provide constant update and read +algorithmic complexity, there is a maximum number (60) of these histogram bins +per shard file. As a trade-off, however, it means not having exact timestamps +for individual changes; a single bin may represent a group of updates. + +Bin sizes increase exponentially the further back in time they go. This means, +for example, that for the recent 24 hours it's possible to target individual +hour blocks; a week back it's only possible to target individual days; and a +few years back can only target individual months. + +How Time Bins Are Updated Over Time +----------------------------------- + +The smallest time granularity is 3 hours. It's not possible to target time +intervals smaller than that. + +A new database starts with an empty list of time bins (``[]``). On the first +update, the first 3 hour bin will be created. + +.. code-block:: text + + [3h] + +If updates continue, after 3 hours there might be a few more ``3h`` bins added + +.. code-block:: text + + [3h, 3h, 3h] + +After about ``60 * 3hs ~= 1 week``, all ``60`` bins would be filled with ``3h`` bins. + +.. code-block:: text + + [3h, 3h, ..., 3h, 3h, 3h] + <--------- 60 ----------> + +Next, some of the oldest bins will start to be merged together to form 6 hour bins: + +.. code-block:: text + + [6h, 6h, ..., 3h, 3h, 3h] + <--------- 60 ----------> + +And after a few years it might look like: + +.. code-block:: text + + [4y, 2y, ... 12h, 6h, 3h] + <--------- 60 ----------> + +``_time_seq`` Endpoint +---------------------- + +The new :ref:`_time_seq ` API endpoint can help inspect the +time histograms for each db shard file. + +**Request**: + +.. code-block:: http + + GET db/_time_seq HTTP/1.1 + Accept: application/json + Host: localhost:5984 + +**Response**: + +.. code-block:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "time_seq": { + "00000000-ffffffff": { + "node1@127.0.0.1": [ + ["2026-01-08T03:00:00Z", 14], + ["2026-01-08T09:00:00Z", 5], + ["2026-01-08T15:00:00Z", 22], + ["2026-01-08T18:00:00Z", 8], + ["2026-01-08T21:00:00Z", 16], + ["2026-01-09T03:00:00Z", 20], + ["2026-01-09T06:00:00Z", 5], + ["2026-01-09T09:00:00Z", 5], + ["2026-01-09T12:00:00Z", 5], + ["2026-01-09T15:00:00Z", 10], + ["2026-01-09T18:00:00Z", 2] + ] + } + } + } + +The times are the start times of each bin, and the values are the number of +changes which occurred in that time interval. For example, in the time bin +started at ``2026-01-08T09:00:00Z`` there were ``5`` changes. + +Since this is a histogram we can pipe it into a Python script with matlotlib and +visualise it. + +.. figure:: ../../../images/time_seq.png + :align: center + + Time seq histogram. + +Times are not exact +------------------- + +If the ``since`` time instant falls in the middle of a time bin interval, then +all the rows from that time bine are returned. In other words, there is a +chance to get some older rows than the provided ``since`` timestamp. If +there is a need to get exact time bounded rows, use a custom time value +per document and a selector filter with the ``since`` timestamp parameter. + +This can be seen in the example above -- the passed in ``since`` value +``2026-01-09T19:01:02Z`` doesn't fall on a bin start. So we got all the rows +from the start of the ``2026-01-09T18:00:00Z`` bin. If we zoom in on the +rendered histogram to the right this might look like this: + +.. figure:: ../../../images/time_seq_zoom.png + :align: center + + Zoom in to last three bins. diff --git a/src/docs/src/api/database/cleanup.rst b/src/docs/src/api/database/cleanup.rst index 6193888e90a..d93eb301a13 100644 --- a/src/docs/src/api/database/cleanup.rst +++ b/src/docs/src/api/database/cleanup.rst @@ -58,6 +58,8 @@ "ok": true } +.. _api/db/nouveau_cleanup: + ========================== ``/{db}/_nouveau_cleanup`` ========================== diff --git a/src/docs/src/config/auth.rst b/src/docs/src/config/auth.rst index 406ab45127c..fac69da7c24 100644 --- a/src/docs/src/config/auth.rst +++ b/src/docs/src/config/auth.rst @@ -329,6 +329,18 @@ Authentication Configuration [chttpd_auth] secret = 92de07df7e7a3fe14808cef90a7cc0d91 + .. note:: + You can change the secret value at any time. New cookies will be + signed with the new value and the previous value will be cached + for the duration of the auth ``timeout`` parameter. + + The secret value should be set on all nodes at the same time. CouchDB + will tolerate a discrepancy, however, as each node sends its secret + to the other nodes of the cluster. + + The easiest rotation method is to enable the config auto-reload + feature then update the secret in the ``.ini`` file of each node. + .. config:option:: timeout :: Session timeout .. versionchanged:: 3.2 moved from [couch_httpd_auth] to [chttpd_auth] section diff --git a/src/docs/src/config/scanner.rst b/src/docs/src/config/scanner.rst index 9c56891d2d3..e03ad7006a9 100644 --- a/src/docs/src/config/scanner.rst +++ b/src/docs/src/config/scanner.rst @@ -263,3 +263,21 @@ settings in their ``[{plugin}]`` section. The database may override this setting with the :ref:`api/db/auto_purge` endpoint. If neither is set, the plugin will not purge deleted documents. + + .. config:option:: log_level + + Set the log level for starting, stopping and purge report summary log entries. :: + + [couch_auto_purge_plugin] + log_level = info + + .. config:option:: dry_run + + When set to ``true`` the plugin does everything (scanning, revision + processing, etc) but skips the actual purge step. Optionally use the + ``log_level`` plugin setting to increase the severity of log reports so + it's clear when the plugin starts, stops and how many revisions it found + to purge. :: + + [couch_auto_purge_plugin] + dry_run = false diff --git a/src/docs/src/ddocs/nouveau.rst b/src/docs/src/ddocs/nouveau.rst index d091ca2e8f7..d7626cfc5df 100644 --- a/src/docs/src/ddocs/nouveau.rst +++ b/src/docs/src/ddocs/nouveau.rst @@ -63,6 +63,39 @@ results from deeper in the result set. A nouveau index will inherit the partitioning type from the ``options.partitioned`` field of the design document that contains it. +.. _ddoc/nouveau/lucene_upgrade: + +Lucene Version Upgrade +====================== + +Nouveau has been upgraded to use Lucene 10, earlier releases used Lucene 9. + +Nouveau can query and update indexes created by Lucene 9 but will not create new +ones. The index definition can optionally define a ``lucene_version`` field +(which must be either 9 or 10 expressed as an integer). If not specified +when defining a new index the current version (10) will be automatically +added to the definition. + +As Lucene only supports indexes up to one major release behind the current, it +is important to rebuild all indexes to the current release. As Lucene major +releases are infrequent, and Nouveau supports 9 and 10 versions simultaneously +it is only necessary to rebuild version 9 indexes before Nouveau upgrades to +Lucene 11 (when it exists). A ``couch_scanner`` plugin is available to +automate this process, and can be enabled as follows; + +.. code-block:: ini + + [couch_scanner_plugins] + nouveau_index_upgrader = true + +The plugin will scan all design documents for index definitions either with no +``lucene_version`` field or one equal to a previous version (lower than +10). The new index will be built by the plugin and, on successful +completion, will update the ``lucene_version`` field in the index +definition. Search requests against that index will seamlessly switch from the +old index to the new one. Invoking the :ref:`_nouveau_cleanup ` +will delete the old indexes. + .. _ddoc/nouveau/field_types: Field Types diff --git a/src/docs/src/install/nouveau.rst b/src/docs/src/install/nouveau.rst index 0dc031914f3..152a2fa2f23 100644 --- a/src/docs/src/install/nouveau.rst +++ b/src/docs/src/install/nouveau.rst @@ -25,7 +25,7 @@ service that embeds `Apache Lucene `_. Typically, thi service is installed on the same host as CouchDB and communicates with it over the loopback network. -Nouveau server is runtime-compatible with Java 11 or higher. +Nouveau server is runtime-compatible with Java 21 or higher. Enable Nouveau ============== diff --git a/src/docs/src/install/troubleshooting.rst b/src/docs/src/install/troubleshooting.rst index f8d13eefd29..c8c87d9a5df 100644 --- a/src/docs/src/install/troubleshooting.rst +++ b/src/docs/src/install/troubleshooting.rst @@ -54,7 +54,7 @@ things: .. code-block:: text %% test SSL support. If this fails, ensure you have the OTP erlang-crypto library installed - crypto:md5_init(). + crypto:hash_init(sha). %% test Snappy compression. If this fails, check your CouchDB configure script output or alternatively %% if your distro comes with erlang-snappy make sure you're using only the CouchDB supplied version @@ -74,9 +74,8 @@ things: Erlang/OTP 17 [erts-6.2] [source] [64-bit] [smp:2:2] [async-threads:10] [kernel-poll:false] Eshell V6.2 (abort with ^G) - 1> crypto:md5_init(). - <<1,35,69,103,137,171,205,239,254,220,186,152,118,84,50, - 16,0,0,0,0,0,0,0,0,0,0,0,0,0,...>> + 1> crypto:hash_init(sha). + #Ref<0.3087268666.1613103106.180459> 2> snappy:compress("gogogogogogogogogogogogogogo"). {ok,<<28,4,103,111,102,2,0>>} 3> jiffy:decode(jiffy:encode(<<"[1,2,3,4,5]">>)). diff --git a/src/docs/src/replication/conflicts.rst b/src/docs/src/replication/conflicts.rst index 704adf62d62..315c88707c1 100644 --- a/src/docs/src/replication/conflicts.rst +++ b/src/docs/src/replication/conflicts.rst @@ -347,7 +347,7 @@ to determine for each document whether it is in a conflicting state: View map functions ================== -Views only get the winning revision of a document. However they do also get a +Views only get the winning revision of a document. However, they do also get a ``_conflicts`` member if there are any conflicting revisions. This means you can write a view whose job is specifically to locate documents with conflicts. Here is a simple map function which achieves this: diff --git a/src/docs/src/replication/protocol.rst b/src/docs/src/replication/protocol.rst index 9f967f6ee70..f361bc334c6 100644 --- a/src/docs/src/replication/protocol.rst +++ b/src/docs/src/replication/protocol.rst @@ -1255,7 +1255,7 @@ Documents-Attachments and may handle it as stream with lesser memory footprint. Content-Length: 87 1. Cook spaghetti - 2. Cook meetballs + 2. Cook meatballs 3. Mix them 4. Add tomato sauce 5. ... @@ -1480,7 +1480,7 @@ one by one without any serialization overhead. Content-Length: 87 1. Cook spaghetti - 2. Cook meetballs + 2. Cook meatballs 3. Mix them 4. Add tomato sauce 5. ... diff --git a/src/docs/src/replication/replicator.rst b/src/docs/src/replication/replicator.rst index 65fd298f57c..3d8b6bf7bab 100644 --- a/src/docs/src/replication/replicator.rst +++ b/src/docs/src/replication/replicator.rst @@ -490,13 +490,13 @@ Server restart When CouchDB is restarted, it checks its ``_replicator`` databases and restarts replications described by documents if they are not already in -in a ``completed`` or ``failed`` state. If they are, they are ignored. +a ``completed`` or ``failed`` state. If they are, they are ignored. Clustering ========== In a cluster, replication jobs are balanced evenly among all the nodes -nodes such that a replication job runs on only one node at a time. +such that a replication job runs on only one node at a time. Every time there is a cluster membership change, that is when nodes are added or removed, as it happens in a rolling reboot, replicator @@ -760,12 +760,12 @@ There are multiple ways to specify usernames and passwords for replication endpo ... } - This is the prefererred format as it allows including characters like ``@``, ``:`` - and others in the username and password fields. + This is the preferred format as it allows including characters like ``@``, + ``:`` and others in the username and password fields. - In the userinfo part of the endpoint URL. This allows for a more compact - endpoint represention however, it prevents using characters like ``@`` and ``:`` - in usernames or passwords: + endpoint representation however, it prevents using characters like ``@`` + and ``:`` in usernames or passwords: .. code-block:: javascript @@ -795,14 +795,14 @@ There are multiple ways to specify usernames and passwords for replication endpo This method has the downside of the going through the extra step of base64 encoding. In addition, it could give the impression that it encrypts or - hides the credentials so it could encourage invadvertent sharing and + hides the credentials so it could encourage inadvertent sharing and leaking credentials. When credentials are provided in multiple forms, they are selected in the following order: - ``"auth": {"basic": {...}}`` object - URL userinfo - - ``"Authorization: Basic ..."`` header. + - ``"Authorization: Basic ..."`` header First, the ``auth`` object is checked, and if credentials are defined there, they are used. If they are not, then URL userinfo is checked. If credentials diff --git a/src/fabric/src/fabric_db_partition_info.erl b/src/fabric/src/fabric_db_partition_info.erl index 2b4685fd21f..824ccce5641 100644 --- a/src/fabric/src/fabric_db_partition_info.erl +++ b/src/fabric/src/fabric_db_partition_info.erl @@ -61,30 +61,33 @@ handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Shard, #acc{} = Acc) -> error -> {error, {nodedown, <<"progress not possible">>}} end; -handle_message({rexi_EXIT, Reason}, Shard, #acc{} = Acc) -> - #acc{counters = Counters, ring_opts = RingOpts} = Acc, +handle_message({rexi_EXIT, Reason}, #shard{dbname = Name} = Shard, #acc{} = Acc) -> + #acc{counters = Counters, ring_opts = RingOpts, replies = Replies} = Acc, NewCounters = fabric_dict:erase(Shard, Counters), case fabric_ring:is_progress_possible(NewCounters, RingOpts) of true -> {ok, Acc#acc{counters = NewCounters}}; false -> - {error, Reason} + case Replies of + [_ | _] -> {stop, format_response(Name, Replies)}; + _ -> {error, Reason} + end end; handle_message({ok, Info}, #shard{dbname = Name} = Shard, #acc{} = Acc) -> #acc{counters = Counters, replies = Replies} = Acc, Replies1 = [Info | Replies], Counters1 = fabric_dict:erase(Shard, Counters), case fabric_dict:size(Counters1) =:= 0 of - true -> - [FirstInfo | RestInfos] = Replies1, - PartitionInfo = get_max_partition_size(FirstInfo, RestInfos), - {stop, [{db_name, Name} | format_partition(PartitionInfo)]}; - false -> - {ok, Acc#acc{counters = Counters1, replies = Replies1}} + true -> {stop, format_response(Name, Replies1)}; + false -> {ok, Acc#acc{counters = Counters1, replies = Replies1}} end; handle_message(_, _, #acc{} = Acc) -> {ok, Acc}. +format_response(DbName, [FirstInfo | RestInfos]) -> + PartitionInfo = get_max_partition_size(FirstInfo, RestInfos), + [{db_name, DbName} | format_partition(PartitionInfo)]. + get_max_partition_size(Max, []) -> Max; get_max_partition_size(MaxInfo, [NextInfo | Rest]) -> @@ -139,6 +142,43 @@ worker_exit_test() -> ?assertEqual({error, bam}, handle_message({rexi_EXIT, bam}, S2, Acc2)). +worker_down_and_mm_test() -> + [S1, S2, S3] = [ + mk_shard("n1", [0, 4]), + mk_shard("n2", [0, 8]), + mk_shard("n3", [0, 4]) + ], + Acc1 = #acc{ + counters = fabric_dict:init([S1, S2, S3], nil), + ring_opts = [{any, [S1, S2, S3]}], + replies = [] + }, + + N1 = S1#shard.node, + {ok, Acc2} = handle_message({rexi_DOWN, nil, {nil, N1}, nil}, nil, Acc1), + + Info = [ + {partition, <<"xx">>}, + {doc_count, 0}, + {doc_del_count, 0}, + {sizes, [{active, 0}, {external, 0}]} + ], + + {ok, Acc3} = handle_message({ok, Info}, S2, Acc2), + + N3 = S3#shard.node, + {stop, Res} = handle_message({rexi_EXIT, {maintenance_mode, N3}}, S3, Acc3), + ?assertMatch( + [ + {db_name, _}, + {sizes, {[{active, 0}, {external, 0}]}}, + {partition, <<"xx">>}, + {doc_count, 0}, + {doc_del_count, 0} + ], + Res + ). + mk_shard(Name, Range) -> Node = list_to_atom(Name), BName = list_to_binary(Name), diff --git a/src/fabric/src/fabric_open_revs.erl b/src/fabric/src/fabric_open_revs.erl index b4f95df8a3d..30ab7c74b17 100644 --- a/src/fabric/src/fabric_open_revs.erl +++ b/src/fabric/src/fabric_open_revs.erl @@ -30,7 +30,8 @@ r, args, reqs, - workers + workers, + errors = [] }). go(_DbName, [], _Options) -> @@ -72,6 +73,9 @@ handle_message({rexi_DOWN, _, {_, NodeRef}, _}, _Worker, #st{} = St) -> Reqs1 = maps:fold(FoldFun, Reqs, DeadWorkers), Error = {error, {nodedown, <<"progress not possible">>}}, handle_error(Error, St#st{workers = Workers1, reqs = Reqs1}); +handle_message({rexi_EXIT, {maintenance_mode, _Node}}, Worker, #st{} = St) -> + % Remove the node to make it easier to de-duplicate later + handle_message(maintenance_mode, Worker, St); handle_message({rexi_EXIT, Reason}, Worker, #st{} = St) -> handle_message(Reason, Worker, St); handle_message({error, Reason}, Worker, #st{} = St) -> @@ -112,7 +116,7 @@ responses_fold({ArgRef, NewResp}, #{} = Reqs) -> } }. -handle_error(Error, #st{workers = Workers, reqs = Reqs} = St) -> +handle_error(Error, #st{workers = Workers, errors = Errors, reqs = Reqs} = St) -> case success_possible(Reqs) of true -> case have_viable_workers(Workers) of @@ -124,7 +128,8 @@ handle_error(Error, #st{workers = Workers, reqs = Reqs} = St) -> end; false -> stop_workers(Workers), - {error, Error} + % We may have multiple errors but need to pick one, so pick the first + {error, hd(merge_errors(Errors, Error))} end. % De-duplicate identical responses as we go along @@ -142,6 +147,17 @@ sort_key({ok, #doc{id = Id, revs = Revs, deleted = Deleted}}) -> sort_key(NotFound) -> NotFound. +% We're trying to hide maintenance mode if possible. So if there are +% non-maintenance mode errors, such as timeouts, etc, we remove the maintenance +% mode from the list, otherwise, we keep it. +% +merge_errors(Errors, Error) -> + Errors1 = lists:uniq([Error | Errors]), + case Errors1 of + [maintenance_mode] -> [maintenance_mode]; + [_ | _] -> lists:delete(maintenance_mode, Errors1) + end. + % Build a #{ArgRef => #req{}} map. ArgRef references the initial {{Id, Revs}, % Opts} tuples and the #req{} is a record keeping track of response for just % that {Id, Revs} pair. @@ -206,11 +222,17 @@ have_viable_workers(#{} = Workers) -> map_size(Workers) > 0. % We can still return success if we either have some waiting workers, or at -% least one response already for each {Id, Revs} pair. +% least one response already for each {Id, Revs} pair. We don't simply check +% for W + R > 0 but check that responses have any entries, as not_found entries +% won't bump the R values. % success_possible(#{} = Reqs) -> - Fun = fun(_, #req{wcnt = W, rcnt = R}, Acc) -> Acc andalso W + R > 0 end, - maps:fold(Fun, true, Reqs). + maps:fold(fun success_possible_fold/3, true, Reqs). + +success_possible_fold(_Key, #req{}, _Acc = false) -> + false; +success_possible_fold(_Key, #req{wcnt = W, responses = Resps}, _Acc) -> + W > 0 orelse Resps =/= []. r_met(#{} = Reqs, ExpectedR) -> Fun = fun(_, #req{rcnt = R}, Acc) -> min(R, Acc) end, @@ -316,6 +338,8 @@ open_revs_quorum_test_() -> ?TDEF_FE(t_finish_quorum), ?TDEF_FE(t_no_quorum_on_different_responses), ?TDEF_FE(t_no_quorum_on_not_found), + ?TDEF_FE(t_not_found_and_maintenance_mode), + ?TDEF_FE(t_all_maintenance_mode), ?TDEF_FE(t_done_on_third), ?TDEF_FE(t_all_different_responses), ?TDEF_FE(t_ancestors_merge_correctly), @@ -400,6 +424,22 @@ t_no_quorum_on_not_found(_) -> Res2 = handle_message([[foo2(), bar1()]], W3, S2), ?assertEqual({stop, [[foo2(), bar1()]]}, Res2). +t_not_found_and_maintenance_mode(_) -> + S0 = #st{workers = Workers0} = st0(), + [W1, W2, W3] = lists:sort(maps:keys(Workers0)), + {ok, S1} = handle_message([[bazNF()]], W1, S0), + {ok, S2} = handle_message({error, timeout}, W2, S1), + Res = handle_message({rexi_EXIT, {maintenance_mode, foo}}, W3, S2), + ?assertEqual({stop, [[bazNF()]]}, Res). + +t_all_maintenance_mode(_) -> + S0 = #st{workers = Workers0} = st0(), + [W1, W2, W3] = lists:sort(maps:keys(Workers0)), + {ok, S1} = handle_message({rexi_EXIT, {maintenance_mode, foo}}, W1, S0), + {ok, S2} = handle_message({rexi_EXIT, {maintenance_mode, foo}}, W2, S1), + Res = handle_message({rexi_EXIT, {maintenance_mode, foo}}, W3, S2), + ?assertEqual({error, maintenance_mode}, Res). + t_done_on_third(_) -> S0 = #st{workers = Workers0} = st0(), [W1, W2, W3] = lists:sort(maps:keys(Workers0)), diff --git a/src/fabric/test/eunit/fabric_tests.erl b/src/fabric/test/eunit/fabric_tests.erl index 77327f4458a..c092f447352 100644 --- a/src/fabric/test/eunit/fabric_tests.erl +++ b/src/fabric/test/eunit/fabric_tests.erl @@ -386,7 +386,11 @@ delete_dbs(DbList) -> setup_fabric() -> Ctx = test_util:start_couch([fabric]), - ok = clear_shards_db(), + ok = + case clear_shards_db() of + ok -> ok; + not_found -> ok + end, Ctx. teardown_fabric(Ctx) -> diff --git a/src/jwtf/src/jwtf.erl b/src/jwtf/src/jwtf.erl index 01f4be3cbae..4c4a3e8ba7f 100644 --- a/src/jwtf/src/jwtf.erl +++ b/src/jwtf/src/jwtf.erl @@ -168,7 +168,8 @@ validate_typ(Props, Checks) -> Required = prop(typ, Checks), TYP = prop(<<"typ">>, Props), case {Required, TYP} of - {undefined, undefined} -> + % ignore unrequired check + {undefined, _} -> ok; {true, undefined} -> throw({bad_request, <<"Missing typ header parameter">>}); diff --git a/src/jwtf/test/jwtf_tests.erl b/src/jwtf/test/jwtf_tests.erl index f7f410e6731..40b190f52ce 100644 --- a/src/jwtf/test/jwtf_tests.erl +++ b/src/jwtf/test/jwtf_tests.erl @@ -88,6 +88,15 @@ invalid_typ_test() -> jwtf:decode(Encoded, [typ], nil) ). +ignored_typ_test() -> + Encoded = encode({[{<<"typ">>, <<"NOPE">>}]}, {[]}), + Ref = make_ref(), + KS = fun(_, _) -> throw(Ref) end, + ?assertEqual( + {error, Ref}, + jwtf:decode(Encoded, [], KS) + ). + missing_alg_test() -> Encoded = encode({[]}, []), ?assertEqual( diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index 0fc92092dd9..726f8d977b6 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -38,7 +38,7 @@ documents_seen }). -create(Db, {Indexes, Trace}, Selector, Opts0) -> +create(Db, {Indexes, Trace}, Selector, Opts) -> Index = case Indexes of [Index0] -> @@ -48,7 +48,6 @@ create(Db, {Indexes, Trace}, Selector, Opts0) -> end, DbName = couch_db:name(Db), - Opts = unpack_bookmark(DbName, Opts0), Stats = mango_execution_stats:stats_init(DbName), DreyfusLimit = get_dreyfus_limit(), @@ -87,9 +86,11 @@ execute(Cursor, UserFun, UserAcc) -> limit = Limit, skip = Skip, selector = Selector, - opts = Opts, + opts = Opts0, execution_stats = Stats } = Cursor, + DbName = couch_db:name(Db), + Opts = unpack_bookmark(DbName, Opts0), Query = mango_selector_text:convert(Selector), QueryArgs = #index_query_args{ q = Query, @@ -387,8 +388,7 @@ t_create_regular(_) -> Indexes = [Index], Trace = #{}, Limit = 10, - Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, bookmark}], - Options1 = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, unpacked_bookmark}], + Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, packed_bookmark}], Db = db, Stats = mango_execution_stats:stats_init(couch_db:name(Db)), Cursor = #cursor{ @@ -397,19 +397,17 @@ t_create_regular(_) -> ranges = null, trace = Trace, selector = selector, - opts = Options1, + opts = Options, limit = Limit, skip = skip, fields = fields, execution_stats = Stats }, - meck:expect(dreyfus_bookmark, unpack, [db_name, bookmark], meck:val(unpacked_bookmark)), ?assertEqual({ok, Cursor}, create(Db, {Indexes, Trace}, selector, Options)). t_create_no_bookmark(_) -> Limit = 99, Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, nil}], - Options1 = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, []}], Db = db, Stats = mango_execution_stats:stats_init(couch_db:name(Db)), Cursor = #cursor{ @@ -418,7 +416,7 @@ t_create_no_bookmark(_) -> ranges = null, trace = trace, selector = selector, - opts = Options1, + opts = Options, limit = Limit, skip = skip, fields = fields, @@ -427,10 +425,23 @@ t_create_no_bookmark(_) -> ?assertEqual({ok, Cursor}, create(Db, {[index], trace}, selector, Options)). t_create_invalid_bookmark(_) -> - Options = [{bookmark, invalid}], - Exception = {mango_error, mango_cursor_text, {invalid_bookmark, invalid}}, - meck:expect(dreyfus_bookmark, unpack, [db_name, invalid], meck:raise(error, something)), - ?assertThrow(Exception, create(db, {[index], trace}, selector, Options)). + Limit = 99, + Options = [{limit, Limit}, {skip, skip}, {fields, fields}, {bookmark, invalid}], + Db = db, + Stats = mango_execution_stats:stats_init(couch_db:name(Db)), + Cursor = #cursor{ + db = Db, + index = index, + ranges = null, + trace = trace, + selector = selector, + opts = Options, + limit = Limit, + skip = skip, + fields = fields, + execution_stats = Stats + }, + ?assertEqual({ok, Cursor}, create(Db, {[index], trace}, selector, Options)). execute_test_() -> { @@ -462,6 +473,16 @@ execute_test_() -> end end ), + meck:expect( + dreyfus_bookmark, + unpack, + fun(db_name, Bookmark) -> + case Bookmark of + [] -> nil; + [bookmark, N] -> [bookmark, N] + end + end + ), meck:expect( dreyfus_bookmark, update, diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 9c5b7a96f7f..d8e39d89248 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -1070,7 +1070,7 @@ check_beginswith(Field, Prefix) -> % in the middle of test output. match_int(mango_selector:normalize(Selector), ?TEST_DOC). -match_beginswith_test() -> +match_beginswith_errors_test() -> % matching ?assertEqual(true, check_beginswith(<<"_id">>, <<"f">>)), % no match (user_id field in the test doc contains an integer) @@ -1087,4 +1087,620 @@ match_beginswith_test() -> check_beginswith(<<"user_id">>, InvalidArg) ). +check_selector(Selector, Results) -> + SelPos = normalize({[{<<"x">>, Selector}]}), + SelNeg = normalize({[{<<"x">>, {[{<<"$not">>, Selector}]}}]}), + + Check = fun({Result, Value}) -> + Doc = {[{<<"x">>, Value}]}, + ?assertEqual(Result, match_int(SelPos, Doc)), + ?assertEqual(not Result, match_int(SelNeg, Doc)) + end, + + lists:foreach(Check, Results). + +match_lt_test() -> + check_selector({[{<<"$lt">>, 5}]}, [{true, 4}, {false, 5}, {false, 6}]), + + check_selector({[{<<"$lt">>, <<"hello">>}]}, [ + {true, <<"held">>}, + {false, <<"hello">>}, + {false, <<"help">>} + ]), + + check_selector({[{<<"$lt">>, [1, 2, 3]}]}, [ + {true, [1, 2, 2]}, + {true, [1, 2]}, + {false, [1, 2, 3]}, + {false, [1, 2, 4]}, + {false, [1, 3]} + ]). + +match_lte_test() -> + check_selector({[{<<"$lte">>, 5}]}, [{true, 4}, {true, 5}, {false, 6}]), + + check_selector({[{<<"$lte">>, <<"hello">>}]}, [ + {true, <<"held">>}, + {true, <<"hello">>}, + {false, <<"help">>} + ]), + + check_selector({[{<<"$lte">>, [1, 2, 3]}]}, [ + {true, [1, 2, 2]}, + {true, [1, 2]}, + {true, [1, 2, 3]}, + {false, [1, 2, 4]}, + {false, [1, 3]} + ]). + +match_gt_test() -> + check_selector({[{<<"$gt">>, 5}]}, [{false, 4}, {false, 5}, {true, 6}]), + + check_selector({[{<<"$gt">>, <<"hello">>}]}, [ + {false, <<"held">>}, + {false, <<"hello">>}, + {true, <<"help">>} + ]), + + check_selector({[{<<"$gt">>, [1, 2, 3]}]}, [ + {false, [1, 2, 2]}, + {false, [1, 2]}, + {false, [1, 2, 3]}, + {true, [1, 2, 4]}, + {true, [1, 3]} + ]). + +match_gte_test() -> + check_selector({[{<<"$gte">>, 5}]}, [{false, 4}, {true, 5}, {true, 6}]), + + check_selector({[{<<"$gte">>, <<"hello">>}]}, [ + {false, <<"held">>}, + {true, <<"hello">>}, + {true, <<"help">>} + ]), + + check_selector({[{<<"$gte">>, [1, 2, 3]}]}, [ + {false, [1, 2, 2]}, + {false, [1, 2]}, + {true, [1, 2, 3]}, + {true, [1, 2, 4]}, + {true, [1, 3]} + ]). + +match_eq_test() -> + check_selector({[{<<"$eq">>, 5}]}, [{true, 5}, {false, 6}]), + check_selector({[{<<"$eq">>, <<"hello">>}]}, [{true, <<"hello">>}, {false, <<"help">>}]), + + check_selector({[{<<"$eq">>, [1, [2, 3, 4], 5]}]}, [ + {true, [1, [2, 3, 4], 5]}, + {false, [1, [2, 3, 4]]}, + {false, [1, [2, 3, 4], 5, 6]}, + {false, [1, [2, 7, 4], 5]} + ]), + + check_selector({[{<<"$eq">>, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}]}, [ + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 8}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"d">>, 7}]}}]}}]}}, + {false, {[{<<"a">>, {[{<<"d">>, {[{<<"c">>, 7}]}}]}}]}} + ]). + +match_ne_test() -> + check_selector({[{<<"$ne">>, 5}]}, [{false, 5}, {true, 6}]), + + % the %ne operator still requires a value to be present... + SelInt = normalize({[{<<"x">>, {[{<<"$ne">>, 5}]}}]}), + ?assertEqual(false, match_int(SelInt, {[]})), + + % ... which, due to normalization, means that using $not with $eq does not + % match the empty doc + SelNotEq = normalize({[{<<"$not">>, {[{<<"x">>, 5}]}}]}), + ?assertEqual(false, match_int(SelNotEq, {[]})), + + check_selector({[{<<"$ne">>, <<"hello">>}]}, [{false, <<"hello">>}, {true, <<"help">>}]), + + check_selector({[{<<"$ne">>, [1, [2, 3, 4], 5]}]}, [ + {false, [1, [2, 3, 4], 5]}, + {true, [1, [2, 3, 4]]}, + {true, [1, [2, 3, 4], 5, 6]}, + {true, [1, [2, 7, 4], 5]} + ]), + + check_selector({[{<<"$ne">>, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}]}, [ + {false, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 7}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"c">>, 8}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"b">>, {[{<<"d">>, 7}]}}]}}]}}, + {true, {[{<<"a">>, {[{<<"d">>, {[{<<"c">>, 7}]}}]}}]}} + ]). + +match_in_test() -> + check_selector({[{<<"$in">>, []}]}, [ + {false, 0}, + {false, true}, + {false, <<"foo">>} + ]), + + check_selector( + {[ + {<<"$in">>, [ + 42, + false, + <<"bar">>, + [[<<"nested">>], <<"list">>], + {[{<<"b">>, 2}]} + ]} + ]}, + [ + {true, 42}, + {true, false}, + {true, <<"bar">>}, + {true, {[{<<"b">>, 2}]}}, + + {false, 43}, + {false, true}, + {false, <<"bars">>}, + {false, {[{<<"b">>, 2}, {<<"c">>, 3}]}}, + + % when the input is an array, $in matches if any of the array items + % match... + {true, [0, 42]}, + {true, [0, false]}, + {true, [0, <<"bar">>]}, + {true, [0, {[{<<"b">>, 2}]}]}, + + % ... which means it doesn't directly match when one of the + % candiate values is itself an array + {false, [[<<"nested">>], <<"list">>]}, + {true, [0, [[<<"nested">>], <<"list">>]]} + ] + ). + +match_nin_test() -> + check_selector({[{<<"$nin">>, []}]}, [ + {true, 0}, + {true, true}, + {true, <<"foo">>} + ]), + + check_selector( + {[ + {<<"$nin">>, [ + 42, + false, + <<"bar">>, + [[<<"nested">>], <<"list">>], + {[{<<"b">>, 2}]} + ]} + ]}, + [ + {false, 42}, + {false, false}, + {false, <<"bar">>}, + {false, {[{<<"b">>, 2}]}}, + + {true, 43}, + {true, true}, + {true, <<"bars">>}, + {true, {[{<<"b">>, 2}, {<<"c">>, 3}]}}, + + % when the input is an array, $nin matches if none of the array items + % match... + {false, [0, 42]}, + {false, [0, false]}, + {false, [0, <<"bar">>]}, + {false, [0, {[{<<"b">>, 2}]}]}, + + % ... which means it doesn't directly match when one of the + % candiate values is itself an array + {true, [[<<"nested">>], <<"list">>]}, + {false, [0, [[<<"nested">>], <<"list">>]]} + ] + ). + +match_all_test() -> + % { "$all": [] } matches nothing, not even arrays + check_selector({[{<<"$all">>, []}]}, [ + {false, []}, + {false, [42]}, + {false, {[]}}, + {false, <<"foo">>} + ]), + + % normally, input lists can contain the required items in any order + check_selector({[{<<"$all">>, [1, 2, 3, 4]}]}, [ + {true, [3, 2, 4, 1]}, + {true, [0, 4, 3, 5, 2, 1, 6]}, + {false, [3, 2, 4]}, + {false, []} + ]), + + % negation means the input must lack at least one of the items + check_selector({[{<<"$not">>, {[{<<"$all">>, [1, 2, 3, 4]}]}}]}, [ + {true, [2, 4, 1]}, + {false, [2, 4, 1, 3]}, + {true, []} + ]), + + % the special $all: [List] form allows the input to exactly match List... + check_selector({[{<<"$all">>, [[1, 2, 3, 4]]}]}, [ + {true, [1, 2, 3, 4]}, + {false, [4, 3, 2, 1]}, + {false, [1, 3, 4]}, + {false, []}, + % ... or to contain List + {true, [5, [1, 2, 3, 4], 6]}, + {false, [5, [1, 3, 4], 6]}, + {false, [5, [1, 3, 2, 4], 6]} + ]), + + % the special behaviour of $all: [X] only applies when X is a list + check_selector({[{<<"$all">>, [<<"hello">>]}]}, [ + {false, <<"hello">>}, + {true, [<<"hello">>]}, + {true, [0, <<"hello">>, 1]}, + {false, []} + ]), + + % values must match exactly and not contain extra fields + check_selector({[{<<"$all">>, [{[{<<"a">>, 1}]}]}]}, [ + {true, [{[{<<"a">>, 1}]}]}, + {false, [{[{<<"a">>, 1}, {<<"b">>, 2}]}]} + ]). + +match_exists_test() -> + check_selector({[{<<"x">>, {[{<<"$exists">>, true}]}}]}, [ + {true, {[{<<"x">>, 0}]}}, + {false, {[{<<"y">>, 0}]}}, + {false, {[]}} + ]), + + check_selector({[{<<"x">>, {[{<<"$exists">>, false}]}}]}, [ + {false, {[{<<"x">>, 0}]}}, + {true, {[{<<"y">>, 0}]}}, + {true, {[]}} + ]), + + % due to normalizing to { "x": { "$ne": 0 } }, this does not match the empty doc + SelNeg = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$eq">>, 0}]}}]}}]}), + SelPos = normalize({[{<<"x">>, 0}]}), + ?assertEqual(false, match_int(SelNeg, {[]})), + ?assertEqual(false, match_int(SelPos, {[]})), + + % including { "$exists": true } in the negated part *does* match the empty doc + check_selector( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$exists">>, true}, + {<<"$eq">>, 0} + ]}} + ]}} + ]}, + [ + {true, {[{<<"x">>, 1}]}}, + {false, {[{<<"x">>, 0}]}}, + {true, {[]}} + ] + ). + +match_type_test() -> + check_selector({[{<<"$type">>, <<"null">>}]}, [ + {true, null}, + {false, false}, + {false, {[]}} + ]), + + check_selector({[{<<"$type">>, <<"boolean">>}]}, [ + {true, true}, + {true, false}, + {false, 0} + ]), + + check_selector({[{<<"$type">>, <<"number">>}]}, [ + {true, 42}, + {true, 3.14}, + {true, 0}, + {false, true}, + {false, [1]}, + {false, <<"1">>} + ]), + + check_selector({[{<<"$type">>, <<"string">>}]}, [ + {true, <<"">>}, + {true, <<"hello">>}, + {false, []} + ]), + + check_selector({[{<<"$type">>, <<"array">>}]}, [ + {true, []}, + {true, [1, 2]}, + {false, {[]}}, + {false, <<"hi">>} + ]), + + check_selector({[{<<"$type">>, <<"object">>}]}, [ + {true, {[]}}, + {true, {[{<<"a">>, 1}]}}, + {false, [{<<"a">>, 1}]}, + {false, null} + ]). + +match_regex_test() -> + check_selector({[{<<"$regex">>, <<"^[0-9a-f]+$">>}]}, [ + {false, <<"">>}, + {true, <<"3a0df5e">>}, + {false, <<"3a0gf5e">>}, + {false, 42} + ]). + +match_beginswith_test() -> + check_selector({[{<<"$beginsWith">>, <<"foo">>}]}, [ + {true, <<"foo">>}, + {true, <<"food">>}, + {true, <<"fool me once">>}, + {false, <<"more food">>}, + {false, <<"fo">>}, + {false, 42} + ]). + +match_mod_test() -> + check_selector({[{<<"$mod">>, [28, 1]}]}, [ + {true, 1}, + {true, 29}, + {true, 57}, + {false, 58}, + {false, <<"57">>} + ]). + +match_size_test() -> + check_selector({[{<<"$size">>, 3}]}, [ + {false, 3}, + {false, <<"fun">>}, + {true, [0, 0, 0]}, + {false, [0, 0]}, + {false, [0, 0, 0, 0]} + ]). + +match_allmatch_test() -> + % $allMatch is defined to return false for empty lists + check_selector({[{<<"$allMatch">>, {[{<<"$eq">>, 0}]}}]}, [ + {false, []}, + {true, [0]}, + {false, [1]}, + {false, [0, 1]} + ]), + + % because of their behaviour on empty lists, { "$not": { "$allMatch": S } } + % is not equivalent to { "$elemMatch": { "$not": S } } + check_selector({[{<<"$elemMatch">>, {[{<<"$ne">>, 0}]}}]}, [ + {false, []}, + {false, [0]}, + {true, [1]}, + {true, [0, 1]} + ]). + +match_elemmatch_test() -> + check_selector({[{<<"$elemMatch">>, {[{<<"$eq">>, 0}]}}]}, [ + {false, []}, + {true, [0]}, + {false, [1]}, + {true, [0, 1]} + ]). + +match_keymapmatch_test() -> + check_selector({[{<<"$keyMapMatch">>, {[{<<"$regex">>, <<"^[a-z]+$">>}]}}]}, [ + {true, {[{<<"hello">>, 0}]}}, + {true, {[{<<"a">>, 1}, {<<"b">>, 2}]}}, + {true, {[{<<"a">>, 1}, {<<"b4">>, 2}]}}, + {false, {[{<<"b4">>, 2}]}}, + {false, {[]}} + ]). + +match_object_test() -> + Doc1 = {[]}, + Doc2 = {[{<<"x">>, {[]}}]}, + Doc3 = {[{<<"x">>, {[{<<"a">>, 1}]}}]}, + Doc4 = {[{<<"x">>, {[{<<"a">>, 1}, {<<"b">>, 2}]}}]}, + Doc5 = {[{<<"x">>, []}]}, + + % the empty selector matches any document + SelEmpty = normalize({[]}), + ?assertEqual({[]}, SelEmpty), + ?assertEqual(true, match_int(SelEmpty, Doc1)), + ?assertEqual(true, match_int(SelEmpty, Doc2)), + ?assertEqual(true, match_int(SelEmpty, Doc3)), + ?assertEqual(true, match_int(SelEmpty, Doc4)), + ?assertEqual(true, match_int(SelEmpty, Doc5)), + + % an inner empty object selector matches only empty objects + SelEmptyField = normalize({[{<<"x">>, {[]}}]}), + ?assertEqual({[{<<"x">>, {[{<<"$eq">>, {[]}}]}}]}, SelEmptyField), + ?assertEqual(false, match_int(SelEmptyField, Doc1)), + ?assertEqual(true, match_int(SelEmptyField, Doc2)), + ?assertEqual(false, match_int(SelEmptyField, Doc3)), + ?assertEqual(false, match_int(SelEmptyField, Doc4)), + ?assertEqual(false, match_int(SelEmptyField, Doc5)), + + % negated empty object selector matches a value which is present and is not the empty object + SelNotEmptyField = normalize({[{<<"$not">>, {[{<<"x">>, {[]}}]}}]}), + ?assertEqual({[{<<"x">>, {[{<<"$ne">>, {[]}}]}}]}, SelNotEmptyField), + ?assertEqual(false, match_int(SelNotEmptyField, Doc1)), + ?assertEqual(false, match_int(SelNotEmptyField, Doc2)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc3)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc4)), + ?assertEqual(true, match_int(SelNotEmptyField, Doc5)), + + % inner object selectors with fields match objects with at least those fields + Sel1Field = normalize({[{<<"x">>, {[{<<"a">>, 1}]}}]}), + ?assertEqual({[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]}, Sel1Field), + ?assertEqual(false, match_int(Sel1Field, Doc1)), + ?assertEqual(false, match_int(Sel1Field, Doc2)), + ?assertEqual(true, match_int(Sel1Field, Doc3)), + ?assertEqual(true, match_int(Sel1Field, Doc4)), + ?assertEqual(false, match_int(Sel1Field, Doc5)), + + % inner object selectors with multiple fields are normalized with $and + Sel2Field = normalize({[{<<"x">>, {[{<<"a">>, 1}, {<<"b">>, 2}]}}]}), + ?assertEqual( + {[ + {<<"$and">>, [ + {[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]}, + {[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]} + ]} + ]}, + Sel2Field + ), + ?assertEqual(false, match_int(Sel2Field, Doc1)), + ?assertEqual(false, match_int(Sel2Field, Doc2)), + ?assertEqual(false, match_int(Sel2Field, Doc3)), + ?assertEqual(true, match_int(Sel2Field, Doc4)), + ?assertEqual(false, match_int(Sel2Field, Doc5)), + + % check shorthand syntax + SelShort = normalize({[{<<"x.b">>, 2}]}), + ?assertEqual({[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]}, SelShort), + ?assertEqual(false, match_int(SelShort, Doc1)), + ?assertEqual(false, match_int(SelShort, Doc2)), + ?assertEqual(false, match_int(SelShort, Doc3)), + ?assertEqual(true, match_int(SelShort, Doc4)), + ?assertEqual(false, match_int(SelShort, Doc5)). + +match_and_test() -> + % $and with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$and">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % due to { "$or": [] } matching anything, negating { "$and": [] } also + % matches anything + SelNotEmpty = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$and">>, []}]}}]}}]}), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, []}]})), + + % and, because { "x": { "$and": [A, B] } } normalizes to + % { "$and": [{ "x": A }, { "x": B }] }, that means + % { "x": { "$not": { "$and": [] } } } normalizes to { "$or": [] }, + % so it matches docs where "x" is not present + ?assertEqual(true, match_int(SelNotEmpty, {[]})), + + % $and with multiple selectors matches if all selectors match + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$and">>, [ + {[{<<"$gt">>, 3}]}, + {[{<<"$lt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})), + + % $not -> $and with multiple selectors matches if any selector does not match + SelNotMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$and">>, [ + {[{<<"$gt">>, 3}]}, + {[{<<"$lt">>, 7}]} + ]} + ]}} + ]}} + ]} + ), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 6}]})), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 2}]})), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelNotMulti, {[]})). + +match_or_test() -> + % $or with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$or">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % similar to $and, due to { "$or": [] } matching anything and our + % normalization rules, negating $or also matches anything + SelNotEmpty = normalize({[{<<"x">>, {[{<<"$not">>, {[{<<"$or">>, []}]}}]}}]}), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelNotEmpty, {[]})), + + % $or with multiple selectors matches if any selector matches + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$or">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})), + + % $not -> $or with multiple selectors matches if no selector matches + SelNotMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$not">>, + {[ + {<<"$or">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelNotMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelNotMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelNotMulti, {[]})). + +match_nor_test() -> + % $nor with an empty array matches anything + SelEmpty = normalize({[{<<"x">>, {[{<<"$nor">>, []}]}}]}), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, 0}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, false}]})), + ?assertEqual(true, match_int(SelEmpty, {[{<<"x">>, []}]})), + ?assertEqual(true, match_int(SelEmpty, {[]})), + + % $nor with multiple selectors matches if no selector matches + SelMulti = normalize( + {[ + {<<"x">>, + {[ + {<<"$nor">>, [ + {[{<<"$lt">>, 3}]}, + {[{<<"$gt">>, 7}]} + ]} + ]}} + ]} + ), + ?assertEqual(true, match_int(SelMulti, {[{<<"x">>, 6}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 2}]})), + ?assertEqual(false, match_int(SelMulti, {[{<<"x">>, 9}]})), + ?assertEqual(false, match_int(SelMulti, {[]})). + -endif. diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py deleted file mode 100644 index cae772b8a85..00000000000 --- a/src/mango/test/02-basic-find-test.py +++ /dev/null @@ -1,320 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - - -import mango -import user_docs - - -class BasicFindTests(mango.UserDocsTests): - def test_bad_selector(self): - bad_selectors = [ - None, - True, - False, - 1.0, - "foobarbaz", - {"foo": {"$not_an_op": 2}}, - {"$gt": 2}, - [None, "bing"], - {"_id": {"": None}}, - ] - for bs in bad_selectors: - try: - self.db.find(bs) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_limit(self): - bad_limits = ([None, True, False, -1, 1.2, "no limit!", {"foo": "bar"}, [2]],) - for bl in bad_limits: - try: - self.db.find({"int": {"$gt": 2}}, limit=bl) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_skip(self): - bad_skips = ([None, True, False, -3, 1.2, "no limit!", {"foo": "bar"}, [2]],) - for bs in bad_skips: - try: - self.db.find({"int": {"$gt": 2}}, skip=bs) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_sort(self): - bad_sorts = ( - [ - None, - True, - False, - 1.2, - "no limit!", - {"foo": "bar"}, - [2], - [{"foo": "asc", "bar": "asc"}], - [{"foo": "asc"}, {"bar": "desc"}], - ], - ) - for bs in bad_sorts: - try: - self.db.find({"int": {"$gt": 2}}, sort=bs) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_fields(self): - bad_fields = ( - [ - None, - True, - False, - 1.2, - "no limit!", - {"foo": "bar"}, - [2], - [[]], - ["foo", 2.0], - ], - ) - for bf in bad_fields: - try: - self.db.find({"int": {"$gt": 2}}, fields=bf) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_r(self): - bad_rs = ([None, True, False, 1.2, "no limit!", {"foo": "bar"}, [2]],) - for br in bad_rs: - try: - self.db.find({"int": {"$gt": 2}}, r=br) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_bad_conflicts(self): - bad_conflicts = ([None, 1.2, "no limit!", {"foo": "bar"}, [2]],) - for bc in bad_conflicts: - try: - self.db.find({"int": {"$gt": 2}}, conflicts=bc) - except Exception as e: - assert e.response.status_code == 400 - else: - raise AssertionError("bad find") - - def test_simple_find(self): - docs = self.db.find({"age": {"$lt": 35}}) - assert len(docs) == 3 - assert docs[0]["user_id"] == 9 - assert docs[1]["user_id"] == 1 - assert docs[2]["user_id"] == 7 - - def test_multi_cond_and(self): - docs = self.db.find({"manager": True, "location.city": "Longbranch"}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 7 - - def test_multi_cond_duplicate_field(self): - # need to explicitly define JSON as dict won't allow duplicate keys - body = ( - '{"selector":{"location.city":{"$regex": "^L+"},' - '"location.city":{"$exists":true}}}' - ) - r = self.db.sess.post(self.db.path("_find"), data=body) - mango.raise_for_status(r) - docs = r.json()["docs"] - - # expectation is that only the second instance - # of the "location.city" field is used - self.assertEqual(len(docs), 15) - - def test_multi_cond_or(self): - docs = self.db.find( - { - "$and": [ - {"age": {"$gte": 75}}, - {"$or": [{"name.first": "Mathis"}, {"name.first": "Whitley"}]}, - ] - } - ) - assert len(docs) == 2 - assert docs[0]["user_id"] == 11 - assert docs[1]["user_id"] == 13 - - def test_multi_col_idx(self): - docs = self.db.find( - { - "location.state": {"$and": [{"$gt": "Hawaii"}, {"$lt": "Maine"}]}, - "location.city": {"$lt": "Longbranch"}, - } - ) - assert len(docs) == 1 - assert docs[0]["user_id"] == 6 - - def test_missing_not_indexed(self): - docs = self.db.find({"favorites.3": "C"}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 6 - - docs = self.db.find({"favorites.3": None}) - assert len(docs) == 0 - - docs = self.db.find({"twitter": {"$gt": None}}) - assert len(docs) == 4 - assert docs[0]["user_id"] == 1 - assert docs[1]["user_id"] == 4 - assert docs[2]["user_id"] == 0 - assert docs[3]["user_id"] == 13 - - def test_limit(self): - docs = self.db.find({"age": {"$gt": 0}}) - assert len(docs) == 15 - for l in [0, 1, 5, 14]: - docs = self.db.find({"age": {"$gt": 0}}, limit=l) - assert len(docs) == l - - def test_skip(self): - docs = self.db.find({"age": {"$gt": 0}}) - assert len(docs) == 15 - for s in [0, 1, 5, 14]: - docs = self.db.find({"age": {"$gt": 0}}, skip=s) - assert len(docs) == (15 - s) - - def test_sort(self): - docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "asc"}]) - docs2 = list(sorted(docs1, key=lambda d: d["age"])) - assert docs1 is not docs2 and docs1 == docs2 - - docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age": "desc"}]) - docs2 = list(reversed(sorted(docs1, key=lambda d: d["age"]))) - assert docs1 is not docs2 and docs1 == docs2 - - def test_sort_desc_complex(self): - docs = self.db.find( - { - "company": {"$lt": "M"}, - "$or": [{"company": "Dreamia"}, {"manager": True}], - }, - sort=[{"company": "desc"}, {"manager": "desc"}], - ) - - companies_returned = list(d["company"] for d in docs) - desc_companies = sorted(companies_returned, reverse=True) - self.assertEqual(desc_companies, companies_returned) - - def test_sort_with_primary_sort_not_in_selector(self): - try: - docs = self.db.find( - {"name.last": {"$lt": "M"}}, sort=[{"name.first": "desc"}] - ) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - resp = e.response.json() - self.assertEqual(resp["error"], "no_usable_index") - else: - raise AssertionError("expected find error") - - def test_sort_exists_true(self): - docs1 = self.db.find( - {"age": {"$gt": 0, "$exists": True}}, sort=[{"age": "asc"}] - ) - docs2 = list(sorted(docs1, key=lambda d: d["age"])) - assert docs1 is not docs2 and docs1 == docs2 - - def test_sort_desc_complex_error(self): - try: - self.db.find( - { - "company": {"$lt": "M"}, - "$or": [{"company": "Dreamia"}, {"manager": True}], - }, - sort=[{"company": "desc"}], - ) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - resp = e.response.json() - self.assertEqual(resp["error"], "no_usable_index") - else: - raise AssertionError("expected find error") - - def test_fields(self): - selector = {"age": {"$gt": 0}} - docs = self.db.find(selector, fields=["user_id", "location.address"]) - for d in docs: - assert sorted(d.keys()) == ["location", "user_id"] - assert sorted(d["location"].keys()) == ["address"] - - def test_r(self): - for r in [1, 2, 3]: - docs = self.db.find({"age": {"$gt": 0}}, r=r) - assert len(docs) == 15 - - def test_empty(self): - docs = self.db.find({}) - # 15 users - assert len(docs) == 15 - - def test_empty_subsel(self): - docs = self.db.find({"_id": {"$gt": None}, "location": {}}) - assert len(docs) == 0 - - def test_empty_subsel_match(self): - self.db.save_docs([{"user_id": "eo", "empty_obj": {}}]) - docs = self.db.find({"_id": {"$gt": None}, "empty_obj": {}}) - assert len(docs) == 1 - assert docs[0]["user_id"] == "eo" - - def test_unsatisfiable_range(self): - docs = self.db.find({"$and": [{"age": {"$gt": 0}}, {"age": {"$lt": 0}}]}) - assert len(docs) == 0 - - def test_explain_view_args(self): - explain = self.db.find({"age": {"$gt": 0}}, fields=["manager"], explain=True) - assert explain["mrargs"]["stable"] == False - assert explain["mrargs"]["update"] == True - assert explain["mrargs"]["reduce"] == False - assert explain["mrargs"]["start_key"] == [0] - assert explain["mrargs"]["end_key"] == [""] - assert explain["mrargs"]["include_docs"] == True - - def test_explain_options(self): - explain = self.db.find({"age": {"$gt": 0}}, fields=["manager"], explain=True) - opts = explain["opts"] - assert opts["r"] == 1 - assert opts["limit"] == 25 - assert opts["skip"] == 0 - assert opts["fields"] == ["manager"] - assert opts["sort"] == {} - assert opts["bookmark"] == "nil" - assert opts["conflicts"] == False - assert opts["execution_stats"] == False - assert opts["partition"] == "" - assert opts["stable"] == False - assert opts["stale"] == False - assert opts["update"] == True - assert opts["use_index"] == [] - assert opts["allow_fallback"] == True - - def test_sort_with_all_docs(self): - explain = self.db.find( - {"_id": {"$gt": 0}, "age": {"$gt": 0}}, sort=["_id"], explain=True - ) - self.assertEqual(explain["index"]["type"], "special") diff --git a/src/mango/test/04-key-tests.py b/src/mango/test/04-key-tests.py deleted file mode 100644 index 1934d4266e0..00000000000 --- a/src/mango/test/04-key-tests.py +++ /dev/null @@ -1,159 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - - -import mango -import unittest - -TEST_DOCS = [ - {"_id": "100", "type": "complex_key", "title": "normal key"}, - { - "_id": "200", - "type": "complex_key", - "title": "key with dot", - "dot.key": "dot's value", - "none": {"dot": "none dot's value"}, - "name.first": "Kvothe", - }, - { - "_id": "300", - "type": "complex_key", - "title": "key with peso", - "$key": "peso", - "deep": {"$key": "deep peso"}, - "name": {"first": "Master Elodin"}, - }, - {"_id": "400", "type": "complex_key", "title": "unicode key", "": "apple"}, - { - "_id": "500", - "title": "internal_fields_format", - "utf8-1[]:string": "string", - "utf8-2[]:boolean[]": True, - "utf8-3[]:number": 9, - "utf8-3[]:null": None, - }, -] - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class KeyTests(mango.DbPerClass): - @classmethod - def setUpClass(klass): - super(KeyTests, klass).setUpClass() - klass.db.save_docs(TEST_DOCS, w=3) - klass.db.create_index(["type"], ddoc="view") - klass.db.create_text_index(ddoc="text") - - def run_check(self, query, check, fields=None, indexes=None): - if indexes is None: - indexes = ["view", "text"] - for idx in indexes: - docs = self.db.find(query, fields=fields, use_index=idx) - check(docs) - - def test_dot_key(self): - query = {"type": "complex_key"} - fields = ["title", "dot\\.key", "none.dot"] - - def check(docs): - assert len(docs) == 4 - assert "dot.key" in docs[1] - assert docs[1]["dot.key"] == "dot's value" - assert "none" in docs[1] - assert docs[1]["none"]["dot"] == "none dot's value" - - self.run_check(query, check, fields=fields) - - def test_peso_key(self): - query = {"type": "complex_key"} - fields = ["title", "$key", "deep.$key"] - - def check(docs): - assert len(docs) == 4 - assert "$key" in docs[2] - assert docs[2]["$key"] == "peso" - assert "deep" in docs[2] - assert docs[2]["deep"]["$key"] == "deep peso" - - self.run_check(query, check, fields=fields) - - def test_unicode_in_fieldname(self): - query = {"type": "complex_key"} - fields = ["title", ""] - - def check(docs): - assert len(docs) == 4 - # note:  == \uf8ff - assert "\uf8ff" in docs[3] - assert docs[3]["\uf8ff"] == "apple" - - self.run_check(query, check, fields=fields) - - # The rest of these tests are only run against the text - # indexes because view indexes don't have to worry about - # field *name* escaping in the index. - - def test_unicode_in_selector_field(self): - query = {"": "apple"} - - def check(docs): - assert len(docs) == 1 - assert docs[0]["\uf8ff"] == "apple" - - self.run_check(query, check, indexes=["text"]) - - def test_internal_field_tests(self): - queries = [ - {"utf8-1[]:string": "string"}, - {"utf8-2[]:boolean[]": True}, - {"utf8-3[]:number": 9}, - {"utf8-3[]:null": None}, - ] - - def check(docs): - assert len(docs) == 1 - assert docs[0]["title"] == "internal_fields_format" - - for query in queries: - self.run_check(query, check, indexes=["text"]) - - def test_escape_period(self): - query = {"name\\.first": "Kvothe"} - - def check(docs): - assert len(docs) == 1 - assert docs[0]["name.first"] == "Kvothe" - - self.run_check(query, check, indexes=["text"]) - - query = {"name.first": "Kvothe"} - - def check_empty(docs): - assert len(docs) == 0 - - self.run_check(query, check_empty, indexes=["text"]) - - def test_object_period(self): - query = {"name.first": "Master Elodin"} - - def check(docs): - assert len(docs) == 1 - assert docs[0]["title"] == "key with peso" - - self.run_check(query, check, indexes=["text"]) - - query = {"name\\.first": "Master Elodin"} - - def check_empty(docs): - assert len(docs) == 0 - - self.run_check(query, check_empty, indexes=["text"]) diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py deleted file mode 100644 index e484781e989..00000000000 --- a/src/mango/test/05-index-selection-test.py +++ /dev/null @@ -1,432 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango -import user_docs -import unittest - - -class IndexSelectionTests: - def test_basic(self): - resp = self.db.find({"age": 123}, explain=True) - self.assertEqual(resp["index"]["type"], "json") - - def test_with_and(self): - resp = self.db.find( - { - "name.first": "Stephanie", - "name.last": "This doesn't have to match anything.", - }, - explain=True, - ) - self.assertEqual(resp["index"]["type"], "json") - - def test_with_nested_and(self): - resp = self.db.find( - {"name.first": {"$gt": "a", "$lt": "z"}, "name.last": "Foo"}, explain=True - ) - self.assertEqual(resp["index"]["type"], "json") - - def test_with_or(self): - ddocid = "_design/company_and_manager" - - resp = self.db.find( - { - "company": {"$gt": "a", "$lt": "z"}, - "$or": [{"manager": "Foo"}, {"manager": "Bar"}], - }, - explain=True, - ) - self.assertEqual(resp["index"]["ddoc"], ddocid) - - def test_use_most_columns(self): - ddocid = "_design/age" - resp = self.db.find( - { - "name.first": "Stephanie", - "name.last": "Something or other", - "age": {"$gt": 1}, - }, - explain=True, - ) - self.assertNotEqual(resp["index"]["ddoc"], ddocid) - - resp = self.db.find( - { - "name.first": "Stephanie", - "name.last": "Something or other", - "age": {"$gt": 1}, - }, - use_index=ddocid, - explain=True, - ) - self.assertEqual(resp["index"]["ddoc"], ddocid) - - def test_no_valid_sort_index(self): - try: - self.db.find({"_id": {"$gt": None}}, sort=["name"], return_raw=True) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("bad find") - - def test_invalid_use_index(self): - # ddoc id for the age index - ddocid = "_design/age" - r = self.db.find({}, use_index=ddocid, return_raw=True) - self.assertEqual( - r["warning"].split("\n")[0].lower(), - "{0} was not used because it does not contain a valid index for this query.".format( - ddocid - ), - ) - - def test_uses_index_when_no_range_or_equals(self): - # index on ["manager"] should be valid because - # selector requires "manager" to exist. The - # selector doesn't narrow the keyrange so it's - # a full index scan - selector = {"manager": {"$exists": True}} - docs = self.db.find(selector) - self.assertEqual(len(docs), 14) - - resp_explain = self.db.find(selector, explain=True) - self.assertEqual(resp_explain["index"]["type"], "json") - - def test_reject_use_index_invalid_fields(self): - ddocid = "_design/company_and_manager" - selector = {"company": "Pharmex"} - r = self.db.find(selector, use_index=ddocid, return_raw=True) - self.assertEqual( - r["warning"].split("\n")[0].lower(), - "{0} was not used because it does not contain a valid index for this query.".format( - ddocid - ), - ) - - # should still return a correct result - for d in r["docs"]: - self.assertEqual(d["company"], "Pharmex") - - def test_reject_use_index_ddoc_and_name_invalid_fields(self): - ddocid = "_design/company_and_manager" - name = "company_and_manager" - selector = {"company": "Pharmex"} - - resp = self.db.find(selector, use_index=[ddocid, name], return_raw=True) - self.assertEqual( - resp["warning"].split("\n")[0].lower(), - "{0}, {1} was not used because it is not a valid index for this query.".format( - ddocid, name - ), - ) - - # should still return a correct result - for d in resp["docs"]: - self.assertEqual(d["company"], "Pharmex") - - def test_reject_use_index_sort_order(self): - # index on ["company","manager"] which should not be valid - # and there is no valid fallback (i.e. an index on ["company"]) - ddocid = "_design/company_and_manager" - selector = {"company": {"$gt": None}} - try: - self.db.find(selector, use_index=ddocid, sort=[{"company": "desc"}]) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("did not reject bad use_index") - - def test_use_index_fallback_if_valid_sort(self): - ddocid_valid = "_design/fallbackfoo" - ddocid_invalid = "_design/fallbackfoobar" - self.db.create_index(fields=["foo"], ddoc=ddocid_invalid) - self.db.create_index(fields=["foo", "bar"], ddoc=ddocid_valid) - selector = {"foo": {"$gt": None}} - - resp_explain = self.db.find( - selector, sort=["foo", "bar"], use_index=ddocid_invalid, explain=True - ) - self.assertEqual(resp_explain["index"]["ddoc"], ddocid_valid) - - resp = self.db.find( - selector, sort=["foo", "bar"], use_index=ddocid_invalid, return_raw=True - ) - self.assertEqual( - resp["warning"].split("\n")[0].lower(), - "{0} was not used because it does not contain a valid index for this query.".format( - ddocid_invalid - ), - ) - self.assertEqual(len(resp["docs"]), 0) - - def test_prefer_use_index_over_optimal_index(self): - # index on ["company"] even though index on ["company", "manager"] is better - ddocid_preferred = "_design/testsuboptimal" - self.db.create_index(fields=["baz"], ddoc=ddocid_preferred) - self.db.create_index(fields=["baz", "bar"]) - selector = {"baz": {"$gt": None}, "bar": {"$gt": None}} - resp = self.db.find(selector, use_index=ddocid_preferred, return_raw=True) - self.assertTrue("warning" not in resp) - - resp_explain = self.db.find(selector, use_index=ddocid_preferred, explain=True) - self.assertEqual(resp_explain["index"]["ddoc"], ddocid_preferred) - - # This doc will not be saved given the new ddoc validation code - # in couch_mrview - def test_manual_bad_view_idx01(self): - design_doc = { - "_id": "_design/bad_view_index", - "language": "query", - "views": { - "queryidx1": { - "map": {"fields": {"age": "asc"}}, - "reduce": "_count", - "options": {"def": {"fields": [{"age": "asc"}]}, "w": 2}, - } - }, - "views": { - "views001": { - "map": "function(employee){if(employee.training)" - + "{emit(employee.number, employee.training);}}" - } - }, - } - with self.assertRaises(KeyError): - self.db.save_doc(design_doc) - - def test_explain_sort_reverse(self): - selector = {"manager": {"$gt": None}} - resp_explain = self.db.find( - selector, fields=["manager"], sort=[{"manager": "desc"}], explain=True - ) - self.assertEqual(resp_explain["index"]["type"], "json") - - def test_use_index_with_invalid_name(self): - for index in ["foo/bar/baz", ["foo", "bar", "baz"]]: - with self.subTest(index=index): - try: - self.db.find({"manager": True}, use_index=index) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("did not fail on invalid index name") - - def test_use_index_without_fallback(self): - with self.subTest(use_index="valid", fallback_available=None): - docs = self.db.find( - {"manager": True}, use_index="manager", allow_fallback=False - ) - assert len(docs) > 0 - - with self.subTest(use_index="invalid", fallback_available=True): - try: - self.db.find( - {"manager": True}, use_index="invalid", allow_fallback=False - ) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("did not fail on invalid index for use_index") - - with self.subTest(use_index="empty", fallback_available=True): - try: - docs = self.db.find( - {"manager": True}, use_index=[], allow_fallback=False - ) - assert len(docs) > 0 - except Exception as e: - raise AssertionError( - "fail due to missing use_index with suitable indexes" - ) - - with self.subTest(use_index="empty", fallback_available=False): - try: - self.db.find({"company": "foobar"}, use_index=[], allow_fallback=False) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError( - "did not fail due to missing use_index without suitable indexes" - ) - - with self.subTest(use_index="invalid", fallback_available=False): - try: - self.db.find( - {"company": "foobar"}, use_index="invalid", allow_fallback=False - ) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("did not fail on invalid index for use_index") - - def test_index_without_fallback(self): - try: - docs = self.db.find({"manager": True}, allow_fallback=False) - assert len(docs) > 0 - except Exception as e: - raise AssertionError("fail on usable indexes") - - def test_no_index_without_fallback(self): - try: - self.db.find({"company": "foobar"}, allow_fallback=False) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - else: - raise AssertionError("did not fail on no usable indexes") - - -class JSONIndexSelectionTests(mango.UserDocsTests, IndexSelectionTests): - @classmethod - def setUpClass(klass): - super(JSONIndexSelectionTests, klass).setUpClass() - - def test_uses_all_docs_when_fields_do_not_match_selector(self): - # index exists on ["company", "manager"] but not ["company"] - # so we should fall back to all docs (so we include docs - # with no "manager" field) - selector = {"company": "Pharmex"} - docs = self.db.find(selector) - self.assertEqual(len(docs), 1) - self.assertEqual(docs[0]["company"], "Pharmex") - self.assertNotIn("manager", docs[0]) - - resp_explain = self.db.find(selector, explain=True) - - self.assertEqual(resp_explain["index"]["type"], "special") - - def test_uses_all_docs_when_selector_doesnt_require_fields_to_exist(self): - # as in test above, use a selector that doesn't overlap with the index - # due to an explicit exists clause - selector = {"company": "Pharmex", "manager": {"$exists": False}} - docs = self.db.find(selector) - self.assertEqual(len(docs), 1) - self.assertEqual(docs[0]["company"], "Pharmex") - self.assertNotIn("manager", docs[0]) - - resp_explain = self.db.find(selector, explain=True) - self.assertEqual(resp_explain["index"]["type"], "special") - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class TextIndexSelectionTests(mango.UserDocsTests): - @classmethod - def setUpClass(klass): - super(TextIndexSelectionTests, klass).setUpClass() - user_docs.add_text_indexes(klass.db, {}) - - def test_with_text(self): - resp = self.db.find( - { - "$text": "Stephanie", - "name.first": "Stephanie", - "name.last": "This doesn't have to match anything.", - }, - explain=True, - ) - self.assertEqual(resp["index"]["type"], "text") - - def test_no_view_index(self): - resp = self.db.find({"name.first": "Ohai!"}, explain=True) - self.assertEqual(resp["index"]["type"], "text") - - def test_with_or(self): - resp = self.db.find( - { - "$or": [ - {"name.first": "Stephanie"}, - {"name.last": "This doesn't have to match anything."}, - ] - }, - explain=True, - ) - self.assertEqual(resp["index"]["type"], "text") - - def test_manual_bad_text_idx(self): - design_doc = { - "_id": "_design/bad_text_index", - "language": "query", - "indexes": { - "text_index": { - "default_analyzer": "keyword", - "default_field": {}, - "selector": {}, - "fields": "all_fields", - "analyzer": { - "name": "perfield", - "default": "keyword", - "fields": {"$default": "standard"}, - }, - } - }, - "indexes": { - "st_index": { - "analyzer": "standard", - "index": 'function(doc){\n index("st_index", doc.geometry);\n}', - } - }, - } - self.db.save_doc(design_doc) - docs = self.db.find({"age": 48}) - self.assertEqual(len(docs), 1) - self.assertEqual(docs[0]["name"]["first"], "Stephanie") - self.assertEqual(docs[0]["age"], 48) - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class MultiTextIndexSelectionTests(mango.UserDocsTests): - @classmethod - def setUpClass(klass): - super(MultiTextIndexSelectionTests, klass).setUpClass() - klass.db.create_text_index(ddoc="foo", analyzer="keyword") - klass.db.create_text_index(ddoc="bar", analyzer="email") - - def test_fallback_to_json_with_multi_text(self): - resp = self.db.find( - {"name.first": "A first name", "name.last": "A last name"}, explain=True - ) - self.assertEqual(resp["index"]["type"], "json") - - def test_multi_text_index_is_error(self): - try: - self.db.find({"$text": "a query"}, explain=True) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - - def test_use_index_works(self): - resp = self.db.find({"$text": "a query"}, use_index="foo", explain=True) - self.assertEqual(resp["index"]["ddoc"], "_design/foo") - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class RegexVsTextIndexTest(mango.DbPerClass): - @classmethod - def setUpClass(klass): - super(RegexVsTextIndexTest, klass).setUpClass() - - def test_regex_works_with_text_index(self): - doc = {"currency": "HUF", "location": "EUROPE"} - self.db.save_docs([doc], w=3) - - selector = {"currency": {"$regex": "HUF"}} - docs = self.db.find(selector) - assert docs == [doc] - - # Now that it is confirmed to be working, try again the - # previous query with a text index on `location`. This - # attempt should succeed as well. - self.db.create_text_index( - name="TextIndexByLocation", fields=[{"name": "location", "type": "string"}] - ) - - docs = self.db.find(selector) - assert docs == [doc] diff --git a/src/mango/test/06-text-default-field-test.py b/src/mango/test/06-text-default-field-test.py deleted file mode 100644 index 1e88967f2e1..00000000000 --- a/src/mango/test/06-text-default-field-test.py +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango -import unittest - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class NoDefaultFieldTest(mango.UserDocsTextTests): - DEFAULT_FIELD = False - - def test_basic(self): - docs = self.db.find({"$text": "Ramona"}) - # Or should this throw an error? - assert len(docs) == 0 - - def test_other_fields_exist(self): - docs = self.db.find({"age": 22}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 9 - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class NoDefaultFieldWithAnalyzer(mango.UserDocsTextTests): - DEFAULT_FIELD = {"enabled": False, "analyzer": "keyword"} - - def test_basic(self): - docs = self.db.find({"$text": "Ramona"}) - assert len(docs) == 0 - - def test_other_fields_exist(self): - docs = self.db.find({"age": 22}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 9 - - -@unittest.skipUnless(mango.has_text_service(), "requires text service") -class DefaultFieldWithCustomAnalyzer(mango.UserDocsTextTests): - DEFAULT_FIELD = {"enabled": True, "analyzer": "keyword"} - - def test_basic(self): - docs = self.db.find({"$text": "Ramona"}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 9 - - def test_not_analyzed(self): - docs = self.db.find({"$text": "Lott Place"}) - assert len(docs) == 1 - assert docs[0]["user_id"] == 9 - - docs = self.db.find({"$text": "Lott"}) - assert len(docs) == 0 - - docs = self.db.find({"$text": "Place"}) - assert len(docs) == 0 diff --git a/src/mango/test/11-ignore-design-docs-test.py b/src/mango/test/11-ignore-design-docs-test.py deleted file mode 100644 index f31dcc5d136..00000000000 --- a/src/mango/test/11-ignore-design-docs-test.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango -import unittest - -DOCS = [ - {"_id": "_design/my-design-doc"}, - {"_id": "54af50626de419f5109c962f", "user_id": 0, "age": 10, "name": "Jimi"}, - {"_id": "54af50622071121b25402dc3", "user_id": 1, "age": 11, "name": "Eddie"}, -] - - -class IgnoreDesignDocsForAllDocsIndexTests(mango.DbPerClass): - def test_should_not_return_design_docs(self): - self.db.save_docs(DOCS) - docs = self.db.find({"_id": {"$gte": None}}) - assert len(docs) == 2 diff --git a/src/mango/test/12-use-correct-index-test.py b/src/mango/test/12-use-correct-index-test.py deleted file mode 100644 index 176835abf1e..00000000000 --- a/src/mango/test/12-use-correct-index-test.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango -import copy - -DOCS = [ - {"_id": "_design/my-design-doc"}, - { - "_id": "54af50626de419f5109c962f", - "user_id": 0, - "age": 10, - "name": "Jimi", - "location": "UK", - "number": 4, - }, - { - "_id": "54af50622071121b25402dc3", - "user_id": 1, - "age": 12, - "name": "Eddie", - "location": "ZAR", - "number": 2, - }, - { - "_id": "54af50622071121b25402dc6", - "user_id": 1, - "age": 6, - "name": "Harry", - "location": "US", - "number": 8, - }, - { - "_id": "54af50622071121b25402dc9", - "name": "Eddie", - "occupation": "engineer", - "number": 7, - }, -] - - -class ChooseCorrectIndexForDocs(mango.DbPerClass): - def setUp(self): - super().setUp(db_per_test=True) - self.db.save_docs(copy.deepcopy(DOCS)) - - def test_choose_index_with_one_field_in_index(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name"], ddoc="zzz") - explain = self.db.find({"name": "Eddie"}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/zzz") - - def test_choose_index_with_two(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") - explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/bbb") - - def test_choose_index_alphabetically(self): - self.db.create_index(["name"], ddoc="aaa") - self.db.create_index(["name"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") - explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/aaa") - - def test_choose_index_most_accurate(self): - self.db.create_index(["name", "age", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") - explain = self.db.find({"name": "Eddie", "age": {"$gte": 12}}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/bbb") - - def test_choose_index_most_accurate_in_memory_selector(self): - self.db.create_index(["name", "location", "user_id"], ddoc="aaa") - self.db.create_index(["name", "age", "user_id"], ddoc="bbb") - self.db.create_index(["name"], ddoc="zzz") - explain = self.db.find({"name": "Eddie", "number": {"$lte": 12}}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/zzz") - - def test_warn_on_full_db_scan(self): - selector = {"not_indexed": "foo"} - explain_resp = self.db.find(selector, explain=True, return_raw=True) - self.assertEqual(explain_resp["index"]["type"], "special") - resp = self.db.find(selector, return_raw=True) - self.assertEqual( - resp["warning"].split("\n")[0].lower(), - "no matching index found, create an index to optimize query time.", - ) - - def test_chooses_idxA(self): - DOCS2 = [{"a": 1, "b": 1, "c": 1}, {"a": 1000, "d": 1000, "e": 1000}] - self.db.save_docs(copy.deepcopy(DOCS2)) - self.db.create_index(["a", "b", "c"]) - self.db.create_index(["a", "d", "e"]) - explain = self.db.find( - {"a": {"$gt": 0}, "b": {"$gt": 0}, "c": {"$gt": 0}}, explain=True - ) - self.assertEqual( - explain["index"]["def"]["fields"], - [{"a": "asc"}, {"b": "asc"}, {"c": "asc"}], - ) - - def test_can_query_with_range_on_secondary_column(self): - self.db.create_index(["age", "name"], ddoc="bbb") - selector = {"age": 10, "name": {"$gte": 0}} - docs = self.db.find(selector) - self.assertEqual(len(docs), 1) - explain = self.db.find(selector, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/bbb") - self.assertEqual(explain["mrargs"]["end_key"], [10, ""]) - - # all documents contain an _id and _rev field they - # should not be used to restrict indexes based on the - # fields required by the selector - def test_choose_index_with_id(self): - self.db.create_index(["name", "_id"], ddoc="aaa") - explain = self.db.find({"name": "Eddie"}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/aaa") - - def test_choose_index_with_rev(self): - self.db.create_index(["name", "_rev"], ddoc="aaa") - explain = self.db.find({"name": "Eddie"}, explain=True) - self.assertEqual(explain["index"]["ddoc"], "_design/aaa") diff --git a/src/mango/test/14-json-pagination-test.py b/src/mango/test/14-json-pagination-test.py deleted file mode 100644 index 7b62f9c61dc..00000000000 --- a/src/mango/test/14-json-pagination-test.py +++ /dev/null @@ -1,269 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango -import copy - -DOCS = [ - {"_id": "100", "name": "Jimi", "location": "AUS", "user_id": 1, "same": "value"}, - {"_id": "200", "name": "Eddie", "location": "BRA", "user_id": 2, "same": "value"}, - {"_id": "300", "name": "Harry", "location": "CAN", "user_id": 3, "same": "value"}, - {"_id": "400", "name": "Eddie", "location": "DEN", "user_id": 4, "same": "value"}, - {"_id": "500", "name": "Jones", "location": "ETH", "user_id": 5, "same": "value"}, - { - "_id": "600", - "name": "Winnifried", - "location": "FRA", - "user_id": 6, - "same": "value", - }, - {"_id": "700", "name": "Marilyn", "location": "GHA", "user_id": 7, "same": "value"}, - {"_id": "800", "name": "Sandra", "location": "ZAR", "user_id": 8, "same": "value"}, -] - - -class PaginateJsonDocs(mango.DbPerClass): - def setUp(self): - super().setUp(db_per_test=True) - self.db.save_docs(copy.deepcopy(DOCS)) - - def test_all_docs_paginate_to_end(self): - selector = {"_id": {"$gt": 0}} - # Page 1 - resp = self.db.find(selector, fields=["_id"], limit=5, return_raw=True) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert docs[0]["_id"] == "100" - assert len(docs) == 5 - - # Page 2 - resp = self.db.find( - selector, fields=["_id"], bookmark=bookmark, limit=5, return_raw=True - ) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert docs[0]["_id"] == "600" - assert len(docs) == 3 - - # Page 3 - resp = self.db.find(selector, bookmark=bookmark, limit=5, return_raw=True) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert len(docs) == 0 - - def test_return_previous_bookmark_for_empty(self): - selector = {"_id": {"$gt": 0}} - # Page 1 - resp = self.db.find(selector, fields=["_id"], return_raw=True) - bookmark1 = resp["bookmark"] - docs = resp["docs"] - assert len(docs) == 8 - - resp = self.db.find( - selector, fields=["_id"], return_raw=True, bookmark=bookmark1 - ) - bookmark2 = resp["bookmark"] - docs = resp["docs"] - assert len(docs) == 0 - - resp = self.db.find( - selector, fields=["_id"], return_raw=True, bookmark=bookmark2 - ) - bookmark3 = resp["bookmark"] - docs = resp["docs"] - assert bookmark3 == bookmark2 - assert len(docs) == 0 - - def test_all_docs_with_skip(self): - selector = {"_id": {"$gt": 0}} - # Page 1 - resp = self.db.find(selector, fields=["_id"], skip=2, limit=5, return_raw=True) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert docs[0]["_id"] == "300" - assert len(docs) == 5 - - # Page 2 - resp = self.db.find( - selector, fields=["_id"], bookmark=bookmark, limit=5, return_raw=True - ) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert docs[0]["_id"] == "800" - assert len(docs) == 1 - resp = self.db.find(selector, bookmark=bookmark, limit=5, return_raw=True) - bookmark = resp["bookmark"] - docs = resp["docs"] - assert len(docs) == 0 - - def test_all_docs_reverse(self): - selector = {"_id": {"$gt": 0}} - resp = self.db.find( - selector, fields=["_id"], sort=[{"_id": "desc"}], limit=5, return_raw=True - ) - docs = resp["docs"] - bookmark1 = resp["bookmark"] - assert len(docs) == 5 - assert docs[0]["_id"] == "800" - - resp = self.db.find( - selector, - fields=["_id"], - sort=[{"_id": "desc"}], - limit=5, - return_raw=True, - bookmark=bookmark1, - ) - docs = resp["docs"] - bookmark2 = resp["bookmark"] - assert len(docs) == 3 - assert docs[0]["_id"] == "300" - - resp = self.db.find( - selector, - fields=["_id"], - sort=[{"_id": "desc"}], - limit=5, - return_raw=True, - bookmark=bookmark2, - ) - docs = resp["docs"] - assert len(docs) == 0 - - def test_bad_bookmark(self): - try: - self.db.find({"_id": {"$gt": 0}}, bookmark="bad-bookmark") - except Exception as e: - resp = e.response.json() - assert resp["error"] == "invalid_bookmark" - assert resp["reason"] == 'Invalid bookmark value: "bad-bookmark"' - assert e.response.status_code == 400 - else: - raise AssertionError("Should have thrown error for bad bookmark") - - def test_throws_error_on_text_bookmark(self): - bookmark = ( - "g2wAAAABaANkABFub2RlMUBjb3VjaGRiLm5ldGwAAAACYQBiP____2poAkY_8AAAAAAAAGEHag" - ) - try: - self.db.find({"_id": {"$gt": 0}}, bookmark=bookmark) - except Exception as e: - resp = e.response.json() - assert resp["error"] == "invalid_bookmark" - assert e.response.status_code == 400 - else: - raise AssertionError("Should have thrown error for bad bookmark") - - def test_index_pagination(self): - self.db.create_index(["location"]) - selector = {"location": {"$gt": "A"}} - resp = self.db.find(selector, fields=["_id"], limit=5, return_raw=True) - docs = resp["docs"] - bookmark1 = resp["bookmark"] - assert len(docs) == 5 - assert docs[0]["_id"] == "100" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark1 - ) - docs = resp["docs"] - bookmark2 = resp["bookmark"] - assert len(docs) == 3 - assert docs[0]["_id"] == "600" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark2 - ) - docs = resp["docs"] - assert len(docs) == 0 - - def test_index_pagination_two_keys(self): - self.db.create_index(["location", "user_id"]) - selector = {"location": {"$gt": "A"}, "user_id": {"$gte": 1}} - resp = self.db.find(selector, fields=["_id"], limit=5, return_raw=True) - docs = resp["docs"] - bookmark1 = resp["bookmark"] - assert len(docs) == 5 - assert docs[0]["_id"] == "100" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark1 - ) - docs = resp["docs"] - bookmark2 = resp["bookmark"] - assert len(docs) == 3 - assert docs[0]["_id"] == "600" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark2 - ) - docs = resp["docs"] - assert len(docs) == 0 - - def test_index_pagination_reverse(self): - self.db.create_index(["location", "user_id"]) - selector = {"location": {"$gt": "A"}, "user_id": {"$gte": 1}} - sort = [{"location": "desc"}, {"user_id": "desc"}] - resp = self.db.find( - selector, fields=["_id"], sort=sort, limit=5, return_raw=True - ) - docs = resp["docs"] - bookmark1 = resp["bookmark"] - assert len(docs) == 5 - assert docs[0]["_id"] == "800" - - resp = self.db.find( - selector, - fields=["_id"], - limit=5, - sort=sort, - return_raw=True, - bookmark=bookmark1, - ) - docs = resp["docs"] - bookmark2 = resp["bookmark"] - assert len(docs) == 3 - assert docs[0]["_id"] == "300" - - resp = self.db.find( - selector, - fields=["_id"], - limit=5, - sort=sort, - return_raw=True, - bookmark=bookmark2, - ) - docs = resp["docs"] - assert len(docs) == 0 - - def test_index_pagination_same_emitted_key(self): - self.db.create_index(["same"]) - selector = {"same": {"$gt": ""}} - resp = self.db.find(selector, fields=["_id"], limit=5, return_raw=True) - docs = resp["docs"] - bookmark1 = resp["bookmark"] - assert len(docs) == 5 - assert docs[0]["_id"] == "100" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark1 - ) - docs = resp["docs"] - bookmark2 = resp["bookmark"] - assert len(docs) == 3 - assert docs[0]["_id"] == "600" - - resp = self.db.find( - selector, fields=["_id"], limit=5, return_raw=True, bookmark=bookmark2 - ) - docs = resp["docs"] - assert len(docs) == 0 diff --git a/src/mango/test/22-covering-index-test.py b/src/mango/test/22-covering-index-test.py deleted file mode 100644 index 9e198761f85..00000000000 --- a/src/mango/test/22-covering-index-test.py +++ /dev/null @@ -1,183 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import mango - - -class CoveringIndexTests: - def test_index_covers_query_1field_index_id(self): - self.is_covered({"age": {"$gte": 32}}, ["_id"], "age") - - def test_index_covers_query_2field_index_id(self): - self.is_covered( - {"company": "Lyria", "manager": True}, ["_id"], "company_and_manager" - ) - - def test_index_covers_query_2field_index_extract_field(self): - self.is_covered( - {"company": {"$exists": True}, "manager": True}, - ["company"], - "company_and_manager", - ) - - def test_index_covers_query_2field_index_extract_field_force_index(self): - self.is_covered( - {"company": {"$exists": True}, "manager": True}, - ["company"], - "company_and_manager", - use_index="company_and_manager", - ) - - def test_index_covers_query_elemMatch(self): - self.is_covered( - {"favorites": {"$elemMatch": {"$eq": "Erlang"}}}, ["favorites"], "favorites" - ) - - def test_index_covers_query_composite_field_id(self): - self.is_covered( - {"name": {"first": "Stephanie", "last": "Kirkland"}}, ["_id"], "name" - ) - - def test_index_does_not_cover_query_empty_selector(self): - self.is_not_covered({}, ["_id"]) - - def test_index_does_not_cover_query_field_not_in_index(self): - self.is_not_covered({"age": {"$gte": 32}}, ["name"]) - - def test_index_does_not_cover_query_all_fields(self): - self.is_not_covered({"age": {"$gte": 32}}, None) - - def test_index_does_not_cover_query_partial_selector_id(self): - self.is_not_covered({"location.state": "Nevada"}, ["_id"]) - - def test_index_does_not_cover_query_partial_selector(self): - self.is_not_covered({"name.last": "Hernandez"}, ["name.first"]) - - def test_index_does_not_cover_selector_with_more_fields(self): - self.is_not_covered( - { - "$and": [ - {"age": {"$ne": 23}}, - {"twitter": {"$not": {"$regex": "^@.*[0-9]+$"}}}, - {"location.address.number": {"$gt": 4288}}, - {"location.city": {"$ne": "Pico Rivera"}}, - ] - }, - ["twitter"], - use_index="twitter", - ) - - -class RegularCoveringIndexTests(mango.UserDocsTests, CoveringIndexTests): - @classmethod - def setUpClass(klass): - super(RegularCoveringIndexTests, klass).setUpClass() - - def is_covered(self, selector, fields, index, use_index=None): - resp = self.db.find(selector, fields=fields, use_index=use_index, explain=True) - self.assertEqual(resp["index"]["type"], "json") - self.assertEqual(resp["index"]["name"], index) - self.assertEqual(resp["mrargs"]["include_docs"], False) - self.assertEqual(resp["covering"], True) - - def is_not_covered(self, selector, fields, use_index=None): - resp = self.db.find(selector, fields=fields, use_index=use_index, explain=True) - self.assertEqual(resp["mrargs"]["include_docs"], True) - self.assertEqual(resp["covering"], False) - - def test_covering_index_provides_correct_answer_2field_index(self): - docs = self.db.find( - {"company": {"$exists": True}, "manager": True}, - sort=[{"company": "asc"}], - fields=["company"], - use_index="company_and_manager", - ) - expected = [ - {"company": "Affluex"}, - {"company": "Globoil"}, - {"company": "Lyria"}, - {"company": "Manglo"}, - {"company": "Myopium"}, - {"company": "Niquent"}, - {"company": "Oulu"}, - {"company": "Prosely"}, - {"company": "Tasmania"}, - {"company": "Zialactic"}, - ] - self.assertEqual(docs, expected) - - def test_covering_index_provides_correct_answer_id(self): - docs = self.db.find({"age": {"$gte": 32}}, fields=["_id"]) - expected = [ - {"_id": "659d0430-b1f4-413a-a6b7-9ea1ef071325"}, - {"_id": "48ca0455-8bd0-473f-9ae2-459e42e3edd1"}, - {"_id": "e900001d-bc48-48a6-9b1a-ac9a1f5d1a03"}, - {"_id": "b31dad3f-ae8b-4f86-8327-dfe8770beb27"}, - {"_id": "71562648-6acb-42bc-a182-df6b1f005b09"}, - {"_id": "c78c529f-0b07-4947-90a6-d6b7ca81da62"}, - {"_id": "8e1c90c0-ac18-4832-8081-40d14325bde0"}, - {"_id": "6c0afcf1-e57e-421d-a03d-0c0717ebf843"}, - {"_id": "5b61abc1-a3d3-4092-b9d7-ced90e675536"}, - {"_id": "a33d5457-741a-4dce-a217-3eab28b24e3e"}, - {"_id": "b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4"}, - {"_id": "b1e70402-8add-4068-af8f-b4f3d0feb049"}, - {"_id": "0461444c-e60a-457d-a4bb-b8d811853f21"}, - ] - self.assertEqual(docs, expected) - - -class PartitionedCoveringIndexTests(mango.PartitionedUserDocsTests, CoveringIndexTests): - @classmethod - def setUpClass(klass): - super(PartitionedCoveringIndexTests, klass).setUpClass() - - def is_covered(self, selector, fields, index, use_index=None): - resp = self.db.find( - selector, fields=fields, use_index=use_index, explain=True, partition="0" - ) - self.assertEqual(resp["index"]["type"], "json") - self.assertEqual(resp["index"]["name"], index) - self.assertEqual(resp["mrargs"]["include_docs"], False) - self.assertEqual(resp["covering"], True) - - def is_not_covered(self, selector, fields, use_index=None): - resp = self.db.find( - selector, fields=fields, use_index=use_index, explain=True, partition="0" - ) - self.assertEqual(resp["mrargs"]["include_docs"], True) - self.assertEqual(resp["covering"], False) - - def test_covering_index_provides_correct_answer_2field_index(self): - docs = self.db.find( - {"company": {"$exists": True}, "manager": True}, - sort=[{"company": "asc"}], - fields=["company"], - use_index="company_and_manager", - partition="0", - ) - expected = [ - {"company": "Manglo"}, - {"company": "Oulu"}, - {"company": "Prosely"}, - {"company": "Tasmania"}, - ] - self.assertEqual(docs, expected) - - def test_covering_index_provides_correct_answer_id(self): - docs = self.db.find({"age": {"$gte": 32}}, fields=["_id"], partition="0") - expected = [ - {"_id": "0:0461444c-e60a-457d-a4bb-b8d811853f21"}, - {"_id": "0:5b61abc1-a3d3-4092-b9d7-ced90e675536"}, - {"_id": "0:71562648-6acb-42bc-a182-df6b1f005b09"}, - {"_id": "0:b31dad3f-ae8b-4f86-8327-dfe8770beb27"}, - ] - self.assertEqual(sorted(docs, key=lambda x: x["_id"]), expected) diff --git a/src/mango/test/25-beginswith-test.py b/src/mango/test/25-beginswith-test.py deleted file mode 100644 index 071cc9a1f30..00000000000 --- a/src/mango/test/25-beginswith-test.py +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -import copy -import mango - -DOCS = [ - {"_id": "aaa", "name": "Jimi", "location": "AUS", "age": 27}, - {"_id": "abc", "name": "Eddie", "location": "AND", "age": 65}, - {"_id": "bbb", "name": "Harry", "location": "CAN", "age": 21}, - {"_id": "ccc", "name": "Eddie", "location": "DEN", "age": 37}, - {"_id": "ddd", "name": "Jones", "location": "ETH", "age": 49}, -] - - -def to_utf8_bytes(list): - return [x.encode() for x in list] - - -class BeginsWithOperator(mango.DbPerClass): - def setUp(self): - super().setUp(db_per_test=True) - self.db.save_docs(copy.deepcopy(DOCS)) - self.db.create_index(["location"]) - self.db.create_index(["name", "location"]) - - def get_mrargs(self, selector, sort=None): - explain = self.db.find(selector, sort=sort, explain=True) - return explain["mrargs"] - - def assertDocIds(self, user_ids, docs): - user_ids_returned = list(d["_id"] for d in docs) - user_ids.sort() - user_ids_returned.sort() - self.assertEqual(user_ids, user_ids_returned) - - def test_basic(self): - docs = self.db.find({"location": {"$beginsWith": "A"}}) - - self.assertEqual(len(docs), 2) - self.assertDocIds(["aaa", "abc"], docs) - - def test_json_range(self): - mrargs = self.get_mrargs({"location": {"$beginsWith": "A"}}) - - self.assertEqual(mrargs["start_key"], ["A"]) - end_key_bytes = to_utf8_bytes(mrargs["end_key"]) - self.assertEqual(end_key_bytes, [b"A\xef\xbf\xbf", b""]) - - def test_compound_key(self): - selector = {"name": "Eddie", "location": {"$beginsWith": "A"}} - mrargs = self.get_mrargs(selector) - - self.assertEqual(mrargs["start_key"], ["Eddie", "A"]) - end_key_bytes = to_utf8_bytes(mrargs["end_key"]) - self.assertEqual(end_key_bytes, [b"Eddie", b"A\xef\xbf\xbf", b""]) - - docs = self.db.find(selector) - self.assertEqual(len(docs), 1) - self.assertDocIds(["abc"], docs) - - def test_sort(self): - selector = {"location": {"$beginsWith": "A"}} - cases = [ - { - "sort": ["location"], - "start_key": [b"A"], - "end_key": [b"A\xef\xbf\xbf", b""], - "direction": "fwd", - }, - { - "sort": [{"location": "desc"}], - "start_key": [b"A\xef\xbf\xbf", b""], - "end_key": [b"A"], - "direction": "rev", - }, - ] - - for case in cases: - with self.subTest(sort=case["sort"]): - mrargs = self.get_mrargs(selector, sort=case["sort"]) - self.assertEqual(to_utf8_bytes(mrargs["start_key"]), case["start_key"]) - self.assertEqual(to_utf8_bytes(mrargs["end_key"]), case["end_key"]) - self.assertEqual(mrargs["direction"], case["direction"]) - - def test_all_docs_range(self): - mrargs = self.get_mrargs({"_id": {"$beginsWith": "a"}}) - - self.assertEqual(mrargs["start_key"], "a") - end_key_bytes = to_utf8_bytes(mrargs["end_key"]) - self.assertEqual(end_key_bytes, [b"a", b"\xef\xbf\xbf"]) - - def test_no_index(self): - selector = {"foo": {"$beginsWith": "a"}} - resp_explain = self.db.find(selector, explain=True) - mrargs = resp_explain["mrargs"] - - self.assertEqual(resp_explain["index"]["type"], "special") - self.assertEqual(mrargs["start_key"], None) - self.assertEqual(mrargs["end_key"], "") - - def test_invalid_operand(self): - try: - self.db.find({"_id": {"$beginsWith": True}}) - except Exception as e: - self.assertEqual(e.response.status_code, 400) - resp = e.response.json() - self.assertEqual(resp["error"], "invalid_operator") - else: - raise AssertionError("expected find error") - - def test_does_not_match_non_string_value(self): - docs = self.db.find({"age": {"$beginsWith": "a"}}) - self.assertEqual(len(docs), 0) - - def test_no_matches(self): - docs = self.db.find({"name": {"$beginsWith": "Z"}}) - self.assertEqual(len(docs), 0) - - def test_case_sensitivity(self): - docs = self.db.find({"name": {"$beginsWith": "j"}}) - self.assertEqual(len(docs), 0) - - docs = self.db.find({"name": {"$beginsWith": "J"}}) - self.assertEqual(len(docs), 2) diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl index f748ff7a0af..6a96ae2a9b8 100644 --- a/src/mem3/src/mem3.erl +++ b/src/mem3/src/mem3.erl @@ -130,8 +130,7 @@ shards_int(DbName, Options) when is_list(DbName) -> shards_int(list_to_binary(DbName), Options); shards_int(DbName, Options) -> Ordered = lists:member(ordered, Options), - ShardDbName = - list_to_binary(config:get("mem3", "shards_db", "_dbs")), + ShardDbName = mem3_sync:shards_db(), case DbName of ShardDbName when Ordered -> %% shard_db is treated as a single sharded db to support calls to db_info @@ -141,7 +140,7 @@ shards_int(DbName, Options) -> node = config:node_name(), name = ShardDbName, dbname = ShardDbName, - range = [0, (2 bsl 31) - 1], + range = [0, ?RING_END], order = undefined } ]; @@ -153,7 +152,7 @@ shards_int(DbName, Options) -> node = config:node_name(), name = ShardDbName, dbname = ShardDbName, - range = [0, (2 bsl 31) - 1] + range = [0, ?RING_END] } ]; _ -> diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl index 38a940e4933..6413c72a4b6 100644 --- a/src/mem3/src/mem3_rep.erl +++ b/src/mem3/src/mem3_rep.erl @@ -20,6 +20,7 @@ make_purge_id/2, verify_purge_checkpoint/2, cleanup_purge_checkpoints/1, + have_all_purge_checkpoints/1, find_source_seq/4, find_split_target_seq/4, local_id_hash/1 @@ -207,11 +208,82 @@ cleanup_purge_checkpoints(Db) -> {stop, Acc} end end, - Opts = [{start_key, list_to_binary(?PURGE_PREFIX)}], - {ok, ToDelete} = couch_db:fold_local_docs(Db, FoldFun, [], Opts), + ToDelete = fold_purge_checkpoints(Db, FoldFun, []), DeleteFun = fun(DocId) -> delete_checkpoint(Db, DocId) end, lists:foreach(DeleteFun, ToDelete). +% Check if we have all the internal replicator purge checkpoints. Call this +% before compaction starts to avoid removing purge infos before the internal +% replicator has managed to create the first checkpoint. +% +have_all_purge_checkpoints(ShardName) when is_binary(ShardName) -> + couch_util:with_db(ShardName, fun(Db) -> have_all_purge_checkpoints(Db) end); +have_all_purge_checkpoints(Db) -> + Shards = shards(couch_db:name(Db)), + ReplicatePurges = config:get_boolean("mem3", "replicate_purges", true), + have_all_purge_checkpoints(ReplicatePurges, Db, Shards). + +have_all_purge_checkpoints(true, Db, [_ | _] = Shards) -> + ShardName = couch_db:name(Db), + UUID = couch_db:get_uuid(Db), + FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) -> + case Id of + <> -> + case verify_checkpoint_shard(Shards, Props) of + true -> + Range = couch_util:get_value(<<"range">>, Props), + TBin = couch_util:get_value(<<"target">>, Props), + TNode = binary_to_existing_atom(TBin, latin1), + {ok, sets:add_element({TNode, Range}, Acc)}; + false -> + {ok, Acc} + end; + _ -> + {stop, Acc} + end + end, + CheckpointSet = fold_purge_checkpoints(Db, FoldFun, couch_util:new_set()), + CopySet = + case ShardName == mem3_sync:shards_db() of + true -> + % We're dealing with the shards db itself. By convention + % mem3:shards/1 returns a single #shard{} record with node = + % node(), name = _dbs, range = [0, ?RING_END] and it should + % replicate in a ring to the dbs copy on "next" node in a ring. + % We push purges to the next node and the previous node pull + % purges from us, so we expect to have only two purge + % checkpoints for the next and previous nodes. + Next = mem3_sync:find_next_node(), + Prev = mem3_sync:find_previous_node(), + % If we're the only node, then next == node() and prev == node() + case Next == config:node_name() of + true -> + couch_util:new_set(); + false -> + couch_util:set_from_list([ + {Next, [0, ?RING_END]}, + {Prev, [0, ?RING_END]} + ]) + end; + false -> + % Keep only shard copies. These are not necessarily ones with a matching + % ranges but also overlapping ranges, since the shards may have been split. + SrcRange = mem3:range(ShardName), + IsCopy = fun(#shard{name = Name, node = Node, range = Range}) -> + (not (Name == ShardName andalso Node == config:node_name())) andalso + mem3_util:range_overlap(SrcRange, Range) + end, + Copies = [{T, R} || #shard{node = T, range = R} = S <- Shards, IsCopy(S)], + couch_util:set_from_list(Copies) + end, + sets:size(sets:subtract(CopySet, CheckpointSet)) == 0; +have_all_purge_checkpoints(false, _Db, _Shards) -> + % If purges are not replicated then we assume we have all (0) checkpoints. + true; +have_all_purge_checkpoints(_, _Db, []) -> + % For a unsharded db we also assume we have all (0) checkpoints. + true. + delete_checkpoint(Db, DocId) -> DbName = couch_db:name(Db), LogMsg = "~p : deleting inactive purge checkpoint ~s : ~s", @@ -229,6 +301,11 @@ delete_checkpoint(Db, DocId) -> ok end. +fold_purge_checkpoints(Db, FoldFun, Acc0) -> + Opts = [{start_key, list_to_binary(?PURGE_PREFIX)}], + {ok, Acc1} = couch_db:fold_local_docs(Db, FoldFun, Acc0, Opts), + Acc1. + verify_purge_checkpoint(DbName, Props) -> try case couch_util:get_value(<<"type">>, Props) of @@ -251,17 +328,33 @@ shards(DbName) -> end. verify_checkpoint_shard(Shards, Props) when is_list(Shards), is_list(Props) -> - Range = couch_util:get_value(<<"range">>, Props), - Fun = fun(S, Acc) -> - case mem3:range(S) == Range of - true -> [mem3:node(S) | Acc]; - false -> Acc - end - end, - Nodes = lists:foldl(Fun, [], Shards), TBin = couch_util:get_value(<<"target">>, Props), TNode = binary_to_existing_atom(TBin, latin1), - lists:member(TNode, Nodes) andalso lists:member(TNode, mem3:nodes()). + ShardsDb = mem3_sync:shards_db(), + case Shards of + [#shard{dbname = ShardsDb}] -> + % This is shards db itself. It's a special case since replications + % copies are other shard db copies replicated in a ring. We push + % purges to the next and node and the previous node pull purges + % from us. So we expect to have two purge replication checkpoints. + Next = mem3_sync:find_next_node(), + Prev = mem3_sync:find_previous_node(), + % If we're the only node, the next == node() + case Next == config:node_name() of + true -> false; + false -> TNode == Next orelse TNode == Prev + end; + _ -> + Range = couch_util:get_value(<<"range">>, Props), + Fun = fun(S, Acc) -> + case mem3:range(S) == Range of + true -> [mem3:node(S) | Acc]; + false -> Acc + end + end, + Nodes = lists:foldl(Fun, [], Shards), + lists:member(TNode, Nodes) andalso lists:member(TNode, mem3:nodes()) + end. %% @doc Find and return the largest update_seq in SourceDb %% that the client has seen from TargetNode. @@ -1232,4 +1325,184 @@ target_not_in_shard_map(_) -> ?assertEqual(1, map_size(Map)), ?assertMatch(#{R0f := #shard{name = Name, node = 'n3'}}, Map). +purge_checkpoints_test_() -> + { + foreach, + fun() -> + Ctx = test_util:start_couch([mem3, fabric]), + config:set("mem3", "replicate_purges", "true", false), + meck:new(mem3, [passthrough]), + meck:new(mem3_sync, [passthrough]), + meck:expect(mem3, nodes, 0, [node(), n2, n3]), + Ctx + end, + fun(Ctx) -> + meck:unload(), + config:delete("mem3", "replicate_purges", false), + test_util:stop_couch(Ctx) + end, + [ + ?TDEF_FE(t_not_sharded), + ?TDEF_FE(t_purges_not_replicated), + ?TDEF_FE(t_have_all_checkpoints), + ?TDEF_FE(t_have_all_shards_db), + ?TDEF_FE(t_verify_checkpoint_shards_db) + ] + }. + +t_not_sharded(_) -> + meck:expect(mem3, shards, 1, meck:raise(error, database_does_not_exist)), + Name = <<"mem3_rep_test", (couch_uuids:random())/binary>>, + {ok, Db} = couch_server:create(Name, [?ADMIN_CTX]), + couch_db:close(Db), + ?assert(have_all_purge_checkpoints(Name)), + ok = couch_server:delete(Name, [?ADMIN_CTX]). + +t_purges_not_replicated(_) -> + R07 = [16#00000000, 16#7fffffff], + R8f = [16#80000000, 16#ffffffff], + R0f = [16#00000000, 16#ffffffff], + + Shards = [ + #shard{node = node(), range = R07}, + #shard{node = node(), range = R8f}, + #shard{node = 'n2', range = R07}, + #shard{node = 'n2', range = R8f}, + #shard{node = 'n3', range = R0f} + ], + meck:expect(mem3, shards, 1, Shards), + + SrcName = <<"shards/00000000-7fffffff/d.1551893550">>, + {ok, Db} = couch_server:create(SrcName, [?ADMIN_CTX]), + couch_db:close(Db), + ?assert(not have_all_purge_checkpoints(SrcName)), + config:set("mem3", "replicate_purges", "false", false), + ?assert(have_all_purge_checkpoints(SrcName)), + ok = couch_server:delete(SrcName, [?ADMIN_CTX]). + +t_have_all_checkpoints(_) -> + R07 = [16#00000000, 16#7fffffff], + R8f = [16#80000000, 16#ffffffff], + R0f = [16#00000000, 16#ffffffff], + + SrcName = <<"shards/00000000-7fffffff/d.1551893551">>, + SrcName1 = <<"shards/80000000-ffffffff/d.1551893551">>, + TgtName1 = <<"shards/00000000-7fffffff/d.1551893551">>, + TgtName2 = <<"shards/80000000-ffffffff/d.1551893551">>, + TgtName3 = <<"shards/00000000-ffffffff/d.1551893551">>, + + Shards = [ + #shard{node = node(), name = SrcName, range = R07}, + #shard{node = node(), name = SrcName1, range = R8f}, + #shard{node = 'n2', name = TgtName1, range = R07}, + #shard{node = 'n2', name = TgtName2, range = R8f}, + #shard{node = 'n3', name = TgtName3, range = R0f} + ], + meck:expect(mem3, shards, 1, Shards), + + Src1 = #shard{name = SrcName, node = node(), range = R07}, + Tgt1 = #shard{name = TgtName1, node = 'n2', range = R07}, + Tgt2 = #shard{name = TgtName2, node = 'n2', range = R8f}, + Tgt3 = #shard{name = TgtName3, node = 'n3', range = R0f}, + + {ok, Db} = couch_server:create(SrcName, [?ADMIN_CTX]), + SrcUuid = couch_db:get_uuid(Db), + + TgtUuid1 = couch_uuids:random(), + % <<"875ce187a5c0f36ee75896d74d10300c">>, + Body1 = purge_cp_body(Src1, Tgt1, 42), + DocId1 = make_purge_id(SrcUuid, TgtUuid1), + Doc1 = #doc{id = DocId1, body = Body1}, + {ok, _} = couch_db:update_doc(Db, Doc1, [?ADMIN_CTX]), + % Not enough checkpoints + ?assert(not have_all_purge_checkpoints(SrcName)), + + Body2 = purge_cp_body(Src1, Tgt2, 43), + TgtUuid2 = couch_uuids:random(), + DocId2 = make_purge_id(SrcUuid, TgtUuid2), + Doc2 = #doc{id = DocId2, body = Body2}, + {ok, _} = couch_db:update_doc(Db, Doc2, [?ADMIN_CTX]), + % Still not enough checkpoints + ?assert(not have_all_purge_checkpoints(SrcName)), + + Body3 = purge_cp_body(Src1, Tgt3, 44), + TgtUuid3 = couch_uuids:random(), + DocId3 = make_purge_id(SrcUuid, TgtUuid3), + Doc3 = #doc{id = DocId3, body = Body3}, + {ok, _} = couch_db:update_doc(Db, Doc3, [?ADMIN_CTX]), + % Now should have all the checkpoints + ?assert(have_all_purge_checkpoints(SrcName)), + + couch_db:close(Db), + ok = couch_server:delete(SrcName, [?ADMIN_CTX]). + +t_have_all_shards_db(_) -> + Dbs = mem3_sync:shards_db(), + {ok, Db} = mem3_util:ensure_exists(Dbs), + SrcUuid = couch_db:get_uuid(Db), + + Range = [0, ?RING_END], + Shards = [ + #shard{node = node(), name = Dbs, dbname = Dbs, range = Range} + ], + meck:expect(mem3, shards, 1, Shards), + + Src1 = #shard{name = Dbs, node = node(), range = Range}, + Tgt1 = #shard{name = Dbs, node = 'n2', range = Range}, + Tgt2 = #shard{name = Dbs, node = 'n3', range = Range}, + + % We're the only node: don't expect any other checkpoints + meck:expect(mem3_sync, find_next_node, 0, node()), + meck:expect(mem3_sync, find_previous_node, 0, node()), + ?assert(have_all_purge_checkpoints(Dbs)), + + % There is another node and we don't have a checkpoint for it + meck:expect(mem3_sync, find_next_node, 0, 'n2'), + meck:expect(mem3_sync, find_previous_node, 0, 'n3'), + ?assert(not have_all_purge_checkpoints(Dbs)), + + Body1 = purge_cp_body(Src1, Tgt1, 42), + TgtUuid1 = couch_uuids:random(), + DocId1 = make_purge_id(SrcUuid, TgtUuid1), + Doc1 = #doc{id = DocId1, body = Body1}, + {ok, _} = couch_db:update_doc(Db, Doc1, [?ADMIN_CTX]), + + % After adding the checkpoint for n2, we should still get false because + % there is no previous checkpoint for n3 pull purges from us + ?assert(not have_all_purge_checkpoints(Dbs)), + + Body2 = purge_cp_body(Src1, Tgt2, 43), + TgtUuid2 = couch_uuids:random(), + DocId2 = make_purge_id(SrcUuid, TgtUuid2), + Doc2 = #doc{id = DocId2, body = Body2}, + {ok, _} = couch_db:update_doc(Db, Doc2, [?ADMIN_CTX]), + + % After adding the checkpoint for n3, we should get true + ?assert(have_all_purge_checkpoints(Dbs)), + + couch_db:close(Db), + ok = couch_server:delete(Dbs, [?ADMIN_CTX]). + +t_verify_checkpoint_shards_db(_) -> + Dbs = mem3_sync:shards_db(), + Range = [0, ?RING_END], + Shards = [ + #shard{node = node(), name = Dbs, dbname = Dbs, range = Range} + ], + Props1 = [ + {<<"target">>, atom_to_binary(n2, latin1)}, + {<<"range">>, Range} + ], + ?assert(not verify_checkpoint_shard(Shards, Props1)), + meck:expect(mem3_sync, find_next_node, 0, 'n2'), + ?assert(verify_checkpoint_shard(Shards, Props1)), + + Props2 = [ + {<<"target">>, atom_to_binary(n3, latin1)}, + {<<"range">>, Range} + ], + ?assert(not verify_checkpoint_shard(Shards, Props2)), + meck:expect(mem3_sync, find_previous_node, 0, 'n3'), + ?assert(verify_checkpoint_shard(Shards, Props2)). + -endif. diff --git a/src/mem3/src/mem3_sync.erl b/src/mem3/src/mem3_sync.erl index 67eb7718167..363b3e0b7b1 100644 --- a/src/mem3/src/mem3_sync.erl +++ b/src/mem3/src/mem3_sync.erl @@ -32,7 +32,8 @@ nodes_db/0, shards_db/0, users_db/0, - find_next_node/0 + find_next_node/0, + find_previous_node/0 ]). -export([ local_dbs/0 @@ -307,10 +308,19 @@ find_next_node() -> Self = node(), LiveNodes = [Self | nodes()], Mem3Nodes = mem3:nodes(), - find_next_node(Self, LiveNodes, Mem3Nodes). + find_next_node(Self, LiveNodes, lists:sort(Mem3Nodes)). -find_next_node(Self, LiveNodes, Mem3Nodes) -> - SortedMem3Nodes = lists:sort(Mem3Nodes), +find_previous_node() -> + Self = node(), + LiveNodes = [Self | nodes()], + Mem3Nodes = mem3:nodes(), + % Previous node is the "next" node in the reverse sorted list + find_previous_node(Self, LiveNodes, lists:sort(Mem3Nodes)). + +find_previous_node(Self, LiveNodes, SortedMem3Nodes) -> + find_next_node(Self, LiveNodes, lists:reverse(SortedMem3Nodes)). + +find_next_node(Self, LiveNodes, SortedMem3Nodes) -> LiveMem3Nodes = [N || N <- SortedMem3Nodes, lists:member(N, LiveNodes)], case LiveMem3Nodes of [] -> @@ -404,13 +414,34 @@ is_job_current(#job{name = Name, node = Node}, ConnectedNodes, Mem3Nodes) -> find_next_node_test() -> ?assertEqual(n, find_next_node(n, [n], [])), + ?assertEqual(n, find_previous_node(n, [], [])), + ?assertEqual(n, find_next_node(n, [n], [n])), + ?assertEqual(n, find_previous_node(n, [n], [n])), + + % We're in the middle ?assertEqual(x, find_next_node(n, [a, n, x], [a, n, x])), + ?assertEqual(a, find_previous_node(n, [a, n, x], [a, n, x])), + + % Two nodes, we're at the end (start, for previous) ?assertEqual(a, find_next_node(n, [a, n], [a, n])), + ?assertEqual(a, find_previous_node(n, [a, n], [a, n])), + + % We're on a node that's not in mem3:nodes() so next/prev is ourselves. ?assertEqual(n, find_next_node(n, [a, n], [a])), - ?assertEqual(x, find_next_node(n, [n, x], [x, n])), + ?assertEqual(n, find_previous_node(n, [a, n], [a])), + + % Two nodes, we're at the start (end, for previous). Live nodes unsorted + ?assertEqual(x, find_next_node(n, [x, n], [n, x])), + ?assertEqual(x, find_previous_node(n, [x, n], [n, x])), + + % Two nodes, we're at the start (end, for previous). Live nodes are sorted ?assertEqual(x, find_next_node(n, [n, x], [n, x])), - ?assertEqual(a, find_next_node(n, [a, n, x], [a, n, y])). + ?assertEqual(x, find_previous_node(n, [n, x], [n, x])), + + % node x is not in mem3:nodes() and node and y is not live + ?assertEqual(a, find_next_node(n, [a, n, x], [a, n, y])), + ?assertEqual(a, find_previous_node(n, [a, n, x], [a, n, y])). is_job_current_test_() -> { diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl index f45ee4063dc..cc728e0ee3d 100644 --- a/src/mem3/src/mem3_util.erl +++ b/src/mem3/src/mem3_util.erl @@ -24,8 +24,7 @@ delete_db_doc/1, shard_info/1, ensure_exists/1, - open_db_doc/1, - update_db_doc/1 + open_db_doc/1 ]). -export([get_or_create_db/2, get_or_create_db_int/2]). -export([is_deleted/1, rotate_list/2]). @@ -134,36 +133,6 @@ write_db_doc(DbName, #doc{id = Id, body = Body} = Doc, ShouldMutate) -> couch_db:close(Db) end. -update_db_doc(Doc) -> - update_db_doc(mem3_sync:shards_db(), Doc, true). - -update_db_doc(DbName, #doc{id = Id, body = Body} = Doc, ShouldMutate) -> - ioq:maybe_set_io_priority({system, DbName}), - {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), - try couch_db:open_doc(Db, Id, [ejson_body]) of - {ok, #doc{body = Body}} -> - % the doc is already in the desired state, we're done here - ok; - {ok, #doc{body = Body1}} -> - % the doc has a new body to be written - {ok, _} = couch_db:update_doc(Db, Doc#doc{body = Body1}, []), - ok; - {not_found, _} when ShouldMutate -> - try couch_db:update_doc(Db, Doc, []) of - {ok, _} -> - ok - catch - conflict -> - % check to see if this was a replication race or a different edit - update_db_doc(DbName, Doc, false) - end; - _ -> - % the doc already exists in a different state - conflict - after - couch_db:close(Db) - end. - -spec range_to_hex([non_neg_integer()]) -> binary(). range_to_hex([B, E]) when is_integer(B), is_integer(E) -> HexB = couch_util:to_hex(<>), diff --git a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl index 140b376358c..799c0721fe4 100644 --- a/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_changes_feed_test.erl @@ -37,7 +37,7 @@ teardown(#{} = Dbs) -> maps:map(fun(_, Db) -> delete_db(Db) end, Dbs). start_couch() -> - test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). + test_util:start_couch([mem3, fabric]). stop_couch(Ctx) -> test_util:stop_couch(Ctx). diff --git a/src/mem3/test/eunit/mem3_reshard_test.erl b/src/mem3/test/eunit/mem3_reshard_test.erl index ec369bfe820..839e7b29319 100644 --- a/src/mem3/test/eunit/mem3_reshard_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_test.erl @@ -39,7 +39,7 @@ teardown(#{} = Dbs) -> meck:unload(). start_couch() -> - test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). + test_util:start_couch([mem3, fabric]). stop_couch(Ctx) -> test_util:stop_couch(Ctx). diff --git a/src/mem3/test/eunit/mem3_seeds_test.erl b/src/mem3/test/eunit/mem3_seeds_test.erl index 4ccae0d07f3..97b1692fa09 100644 --- a/src/mem3/test/eunit/mem3_seeds_test.erl +++ b/src/mem3/test/eunit/mem3_seeds_test.erl @@ -106,7 +106,7 @@ teardown(Ctx) -> catch application:stop(mem3), config:delete("cluster", "seedlist", false), Filename = config:get("mem3", "nodes_db", "_nodes") ++ ".couch", - file:delete(filename:join([?BUILDDIR(), "tmp", "data", Filename])), + file:delete(filename:join([?BUILDDIR(), "data", Filename])), case config:get("couch_httpd_auth", "authentication_db") of undefined -> ok; DbName -> couch_server:delete(list_to_binary(DbName), []) diff --git a/src/mem3/test/eunit/mem3_shards_test.erl b/src/mem3/test/eunit/mem3_shards_test.erl index 14f4bc08466..a42296f2ec3 100644 --- a/src/mem3/test/eunit/mem3_shards_test.erl +++ b/src/mem3/test/eunit/mem3_shards_test.erl @@ -15,10 +15,7 @@ -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). -include_lib("mem3/src/mem3_reshard.hrl"). -% for all_docs function --include_lib("couch_mrview/include/couch_mrview.hrl"). --define(ID, <<"_id">>). -define(TIMEOUT, 60). setup() -> @@ -32,7 +29,7 @@ teardown(#{dbname := DbName}) -> delete_db(DbName). start_couch() -> - test_util:start_couch(?CONFIG_CHAIN, [mem3, fabric]). + test_util:start_couch([mem3, fabric]). stop_couch(Ctx) -> test_util:stop_couch(Ctx). @@ -69,7 +66,7 @@ partitioned_shards_recreated_properly(#{dbname := DbName, dbdoc := DbDoc}) -> ?assert(is_partitioned(Shards)), ok = with_proc(fun() -> couch_server:delete(ShardName, []) end), ?assertThrow({not_found, no_db_file}, is_partitioned(Shard)), - ok = mem3_util:update_db_doc(DbDoc#doc{body = {Body1}}), + {ok, {2, _Rev}} = mem3:update_db_doc(DbDoc#doc{body = {Body1}}), Shards = [Shard | _] = test_util:wait_value( fun() -> diff --git a/src/mem3/test/eunit/mem3_zone_test.erl b/src/mem3/test/eunit/mem3_zone_test.erl index 55fef3342f9..12ec4ad36e2 100644 --- a/src/mem3/test/eunit/mem3_zone_test.erl +++ b/src/mem3/test/eunit/mem3_zone_test.erl @@ -68,7 +68,7 @@ teardown(Ctx) -> catch application:stop(mem3), os:unsetenv("COUCHDB_ZONE"), Filename = config:get("mem3", "nodes_db", "_nodes") ++ ".couch", - file:delete(filename:join([?BUILDDIR(), "tmp", "data", Filename])), + file:delete(filename:join([?BUILDDIR(), "data", Filename])), case config:get("couch_httpd_auth", "authentication_db") of undefined -> ok; DbName -> couch_server:delete(list_to_binary(DbName), []) diff --git a/src/nouveau/include/nouveau.hrl b/src/nouveau/include/nouveau.hrl index e50cd45d3a9..efdc6eb7e62 100644 --- a/src/nouveau/include/nouveau.hrl +++ b/src/nouveau/include/nouveau.hrl @@ -11,9 +11,13 @@ %% See the License for the specific language governing permissions and %% limitations under the License. +-define(LEGACY_LUCENE_VERSION, 9). +-define(TARGET_LUCENE_VERSION, 10). + -record(index, { dbname, ddoc_id, + lucene_version, default_analyzer, field_analyzers, def, diff --git a/src/nouveau/src/nouveau_api.erl b/src/nouveau/src/nouveau_api.erl index cfc88af4f7e..2d140e5806a 100644 --- a/src/nouveau/src/nouveau_api.erl +++ b/src/nouveau/src/nouveau_api.erl @@ -29,6 +29,7 @@ search/2, set_purge_seq/3, set_update_seq/3, + supported_lucene_versions/0, jaxrs_error/2 ]). @@ -214,6 +215,18 @@ set_seq(#index{} = Index, ReqBody) -> send_error(Reason) end. +supported_lucene_versions() -> + Resp = send_if_enabled(<<"/">>, [], <<"GET">>), + case Resp of + {ok, 200, _, RespBody} -> + Json = jiffy:decode(RespBody, [return_maps]), + {ok, maps:get(<<"supported_lucene_versions">>, Json, [])}; + {ok, StatusCode, _, RespBody} -> + {error, jaxrs_error(StatusCode, RespBody)}; + {error, Reason} -> + send_error(Reason) + end. + %% private functions index_path(Path) when is_binary(Path) -> diff --git a/src/nouveau/src/nouveau_fabric_search.erl b/src/nouveau/src/nouveau_fabric_search.erl index 6a47cb95457..06617146596 100644 --- a/src/nouveau/src/nouveau_fabric_search.erl +++ b/src/nouveau/src/nouveau_fabric_search.erl @@ -15,7 +15,7 @@ -module(nouveau_fabric_search). --export([go/4]). +-export([go/3, go/4]). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -38,12 +38,12 @@ go(DbName, GroupId, IndexName, QueryArgs0) when is_binary(GroupId) -> go(DbName, #doc{} = DDoc, IndexName, QueryArgs0) -> case nouveau_util:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> - go(DbName, DDoc, IndexName, QueryArgs0, Index); + go(DbName, QueryArgs0, Index); {error, Reason} -> {error, Reason} end. -go(DbName, #doc{} = _DDoc, _IndexName, QueryArgs0, Index) -> +go(DbName, QueryArgs0, Index) -> Shards = get_shards(DbName, QueryArgs0), {PackedBookmark, #{limit := Limit, sort := Sort} = QueryArgs1} = maps:take(bookmark, QueryArgs0), diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl index 3952a893f24..4bfea753a15 100644 --- a/src/nouveau/src/nouveau_index_updater.erl +++ b/src/nouveau/src/nouveau_index_updater.erl @@ -204,6 +204,7 @@ get_db_info(#index{} = Index) -> index_definition(#index{} = Index) -> #{ + <<"lucene_version">> => Index#index.lucene_version, <<"default_analyzer">> => Index#index.default_analyzer, <<"field_analyzers">> => Index#index.field_analyzers }. diff --git a/src/nouveau/src/nouveau_index_upgrader.erl b/src/nouveau/src/nouveau_index_upgrader.erl new file mode 100644 index 00000000000..9f757cf5577 --- /dev/null +++ b/src/nouveau/src/nouveau_index_upgrader.erl @@ -0,0 +1,165 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader). +-behaviour(couch_scanner_plugin). + +-export([ + start/2, + resume/2, + complete/1, + checkpoint/1, + db/2, + ddoc/3 +]). + +-include("nouveau.hrl"). +-include_lib("couch_scanner/include/couch_scanner_plugin.hrl"). + +start(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Starting.", [], St), + {ok, St}; + false -> + ?INFO("Not starting.", [], St), + skip + end. + +resume(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Resuming.", [], St), + {ok, St}; + false -> + ?INFO("Not resuming.", [], St), + skip + end. + +complete(St) -> + ?INFO("Completed", [], St), + {ok, #{}}. + +checkpoint(_St) -> + {ok, #{}}. + +db(St, _DbName) -> + {ok, St}. + +ddoc(St, _DbName, #doc{id = <<"_design/_", _/binary>>}) -> + {ok, St}; +ddoc(St, DbName, #doc{} = DDoc0) -> + case update_ddoc_versions(DDoc0) of + DDoc0 -> + ok; + DDoc1 -> + Indexes = nouveau_util:design_doc_to_indexes(DbName, DDoc1), + case upgrade_indexes(DbName, Indexes) of + true -> + save_ddoc(DbName, DDoc1); + false -> + ok + end + end, + {ok, St}. + +upgrade_indexes(_DbName, []) -> + true; +upgrade_indexes(DbName, [Index | Rest]) -> + case upgrade_index(DbName, Index) of + true -> + upgrade_indexes(DbName, Rest); + false -> + false + end. + +upgrade_index(DbName, #index{} = Index) -> + ?INFO("Upgrading ~s/~s/~s to version ~B", [ + DbName, + Index#index.ddoc_id, + Index#index.name, + ?TARGET_LUCENE_VERSION + ]), + case + nouveau_fabric_search:go( + DbName, + #{query => <<"*:*">>, bookmark => null, sort => null, limit => 1}, + Index#index{lucene_version = ?TARGET_LUCENE_VERSION} + ) + of + {ok, _SearchResults} -> + true; + {error, _Reason} -> + false + end. + +update_ddoc_versions(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun update_version/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +save_ddoc(DbName, #doc{} = DDoc) -> + {Pid, Ref} = spawn_monitor(fun() -> + case fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]) of + {ok, _} -> + exit(ok); + Else -> + exit(Else) + end + end), + receive + {'DOWN', Ref, process, Pid, ok} -> + ?INFO( + "Updated ~s/~s indexes to version ~B", [DbName, DDoc#doc.id, ?TARGET_LUCENE_VERSION] + ); + {'DOWN', Ref, process, Pid, Else} -> + ?INFO("Failed to update ~s/~s for reason ~p", [DbName, DDoc#doc.id, Else]) + end. + +update_version({IndexName, {Index}}) -> + {IndexName, {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}. + +init_config(ScanId) -> + #{sid => ScanId}. + +should_run(St) -> + couch_scanner_util:on_first_node() andalso upgrade_supported(St). + +upgrade_supported(St) -> + case nouveau_api:supported_lucene_versions() of + {ok, Versions} -> + case lists:member(?TARGET_LUCENE_VERSION, Versions) of + true -> + ?INFO( + "Nouveau server supports upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + true; + false -> + ?WARN( + "Nouveau server does not support upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + false + end; + {error, Reason} -> + ?ERR( + "Nouveau server upgrade check failed for reason ~p", [Reason], St + ), + false + end. diff --git a/src/nouveau/src/nouveau_plugin_couch_db.erl b/src/nouveau/src/nouveau_plugin_couch_db.erl index dcd3ae1f156..d10d7adfc4d 100644 --- a/src/nouveau/src/nouveau_plugin_couch_db.erl +++ b/src/nouveau/src/nouveau_plugin_couch_db.erl @@ -13,10 +13,46 @@ -module(nouveau_plugin_couch_db). -export([ + before_doc_update/3, is_valid_purge_client/2, on_compact/2 ]). +-include("nouveau.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +%% New index definitions get an explicit lucene version property, if missing. +before_doc_update( + #doc{id = <>, revs = {0, []}} = Doc, + Db, + ?INTERACTIVE_EDIT = UpdateType +) -> + #doc{body = {Fields}} = Doc, + case couch_util:get_value(<<"nouveau">>, Fields) of + {Indexes} when is_list(Indexes) -> + [add_versions_to_doc(Doc), Db, UpdateType]; + _ -> + [Doc, Db, UpdateType] + end; +before_doc_update(Doc, Db, UpdateType) -> + [Doc, Db, UpdateType]. + +add_versions_to_doc(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun add_version_to_index/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +add_version_to_index({IndexName, {Index}}) -> + case couch_util:get_value(<<"lucene_version">>, Index) of + undefined -> + {IndexName, + {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}; + _ -> + {IndexName, {Index}} + end. + is_valid_purge_client(DbName, Props) -> nouveau_util:verify_index_exists(DbName, Props). diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index 3df43f2ffcf..f51742d3e4a 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -69,23 +69,33 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> false -> {error, {not_found, <>}}; {IndexName, {Index}} -> + LuceneVersion = couch_util:get_value( + <<"lucene_version">>, Index, ?LEGACY_LUCENE_VERSION + ), DefaultAnalyzer = couch_util:get_value(<<"default_analyzer">>, Index, <<"standard">>), FieldAnalyzers = couch_util:get_value(<<"field_analyzers">>, Index, #{}), case couch_util:get_value(<<"index">>, Index) of undefined -> {error, InvalidDDocError}; Def -> - Sig = - couch_util:to_hex_bin( - crypto:hash( - sha256, - ?term_to_bin( - {DefaultAnalyzer, FieldAnalyzers, Def} - ) + SigTerm = + case LuceneVersion of + ?LEGACY_LUCENE_VERSION -> + {DefaultAnalyzer, FieldAnalyzers, Def}; + _ -> + {LuceneVersion, DefaultAnalyzer, FieldAnalyzers, Def} + end, + Sig = couch_util:to_hex_bin( + crypto:hash( + sha256, + ?term_to_bin( + SigTerm ) - ), + ) + ), {ok, #index{ dbname = DbName, + lucene_version = LuceneVersion, default_analyzer = DefaultAnalyzer, field_analyzers = FieldAnalyzers, ddoc_id = Id, diff --git a/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl new file mode 100644 index 00000000000..d37a1d0071b --- /dev/null +++ b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl @@ -0,0 +1,131 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). +-include_lib("nouveau/include/nouveau.hrl"). + +-define(PLUGIN, nouveau_index_upgrader). + +nouveau_index_upgrader_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_upgrade_legacy_index, 10), + ?TDEF_FE(t_dont_upgrade_latest_index, 10) + ] + }. + +setup() -> + {module, _} = code:ensure_loaded(?PLUGIN), + meck:new(?PLUGIN, [passthrough]), + meck:new(couch_scanner_server, [passthrough]), + meck:new(couch_scanner_util, [passthrough]), + meck:new(nouveau_api, [passthrough]), + meck:expect(nouveau_api, supported_lucene_versions, fun() -> + {ok, [?LEGACY_LUCENE_VERSION, ?TARGET_LUCENE_VERSION]} + end), + Ctx = test_util:start_couch([fabric, couch_scanner]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, "2"}, {n, "1"}]), + config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false), + reset_stats(), + {Ctx, DbName}. + +teardown({Ctx, DbName}) -> + config_delete_section("couch_scanner"), + config_delete_section("couch_scanner_plugins"), + config_delete_section(atom_to_list(?PLUGIN)), + couch_scanner:reset_checkpoints(), + couch_scanner:resume(), + fabric:delete_db(DbName), + test_util:stop_couch(Ctx), + meck:unload(). + +t_upgrade_legacy_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?LEGACY_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + meck:new(nouveau_fabric_search, [passthrough]), + meck:expect(nouveau_fabric_search, go, fun(_DbName, _Args, _Index) -> {ok, []} end), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +t_dont_upgrade_latest_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?TARGET_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +reset_stats() -> + Counters = [ + [couchdb, query_server, process_error_exits], + [couchdb, query_server, process_errors], + [couchdb, query_server, process_exits] + ], + [reset_counter(C) || C <- Counters]. + +reset_counter(Counter) -> + case couch_stats:sample(Counter) of + 0 -> + ok; + N when is_integer(N), N > 0 -> + couch_stats:decrement_counter(Counter, N) + end. + +config_delete_section(Section) -> + [config:delete(K, V, false) || {K, V} <- config:get(Section)]. + +add_ddoc(DbName, DDocId, IndexName, LuceneVersion) -> + {ok, _} = fabric:update_doc(DbName, mkddoc(DDocId, IndexName, LuceneVersion), [?ADMIN_CTX]), + ok. + +get_lucene_version(DbName, DDocId, IndexName) -> + {ok, #doc{body = {Props}}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]), + {Indexes} = couch_util:get_value(<<"nouveau">>, Props), + {Index} = couch_util:get_value(IndexName, Indexes), + couch_util:get_value(<<"lucene_version">>, Index). + +mkddoc(DocId, IndexName, LuceneVersion) -> + Body = #{ + <<"_id">> => DocId, + <<"nouveau">> => #{ + IndexName => #{ + <<"lucene_version">> => LuceneVersion, + <<"index">> => <<"function(doc){}">> + } + } + }, + jiffy:decode(jiffy:encode(Body)). + +num_calls(Fun, Args) -> + meck:num_calls(?PLUGIN, Fun, Args). + +wait_exit(MSec) -> + meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'], MSec). diff --git a/src/rexi/src/rexi_buffer.erl b/src/rexi/src/rexi_buffer.erl index 73bf3dfaef8..90f8bf2e676 100644 --- a/src/rexi/src/rexi_buffer.erl +++ b/src/rexi/src/rexi_buffer.erl @@ -42,7 +42,8 @@ }). start_link(ServerId) -> - gen_server:start_link({local, ServerId}, ?MODULE, [ServerId], []). + GenOpts = couch_util:hibernate_after(?MODULE), + gen_server:start_link({local, ServerId}, ?MODULE, [ServerId], GenOpts). send(Dest, Msg) -> Server = list_to_atom(lists:concat([rexi_buffer, "_", get_node(Dest)])), @@ -90,9 +91,10 @@ handle_info(timeout, #state{sender = nil, count = C} = State) when C > 0 -> counters:add(Counter, 1, -1), case erlang:send(Dest, Msg, [noconnect, nosuspend]) of ok when C =:= 1 -> - % We just sent the last queued messsage, we'll use this opportunity - % to hibernate the process and run a garbage collection - {noreply, NewState, hibernate}; + % We just sent the last queued messsage. If we stay idle for a + % while, as configured via couch_util:hibernate_after/2 we'll go + % into hibernation. + {noreply, NewState}; ok when C > 1 -> % Use a zero timeout to recurse into this handler ASAP {noreply, NewState, 0}; diff --git a/src/smoosh/src/smoosh_persist.erl b/src/smoosh/src/smoosh_persist.erl index f615fcbb93f..af9ea8b4285 100644 --- a/src/smoosh/src/smoosh_persist.erl +++ b/src/smoosh/src/smoosh_persist.erl @@ -194,24 +194,39 @@ t_corrupted_read(_) -> ?assertEqual({error, enoent}, file:read_file_info(Path)). t_check_setup(_) -> - ?assertEqual(disabled, check_setup()), - - meck:expect(config, get_boolean, fun("smoosh", "persist", _) -> true end), - ?assertEqual(ok, check_setup()), - - TDir = ?tempfile(), - meck:expect(config, get, fun("smoosh", "state_dir", _) -> TDir end), - ?assertEqual({error, {"read", enoent}}, check_setup()), - - Dir = state_dir(), - ok = file:make_dir(Dir), - % Can't write, only read - ok = file:change_mode(Dir, 8#500), - ?assertEqual({error, {"write", eacces}}, check_setup()), - % Can't read, only write - ok = file:change_mode(Dir, 8#300), - ?assertEqual({error, {"read access", write}}, check_setup()), - ok = file:del_dir_r(Dir). + % SKIP_TEST_ON_WINDOWS + {Osfamily, _} = os:type(), + case Osfamily of + win32 -> + ok; + _ -> + ?assertEqual(disabled, check_setup()), + + meck:expect(config, get_boolean, fun("smoosh", "persist", _) -> true end), + ?assertEqual(ok, check_setup()), + + TDir = ?tempfile(), + meck:expect(config, get, fun("smoosh", "state_dir", _) -> TDir end), + ?assertEqual({error, {"read", enoent}}, check_setup()), + Dir = state_dir(), + ok = file:make_dir(Dir), + % Can't write, only read. This works only when running as non-admin/non-root. + ok = file:change_mode(Dir, 8#500), + % Did the setup work? + {ok, #file_info{access = Access}} = file:read_file_info(Dir), + case Access of + read -> + ?assertEqual({error, {"write", eacces}}, check_setup()), + % Can't read, only write + ok = file:change_mode(Dir, 8#300), + ?assertEqual({error, {"read access", write}}, check_setup()); + read_write -> + % We must be running as root/admin and we can't really make Dir + % unreadable for ourselves, so skip this part. + ok + end, + ok = file:del_dir_r(Dir) + end. t_persist_unpersist_disabled(_) -> Name = "chan1", @@ -271,15 +286,28 @@ t_persist_unpersist_errors(_) -> Dir = state_dir(), ok = file:make_dir(Dir), - - % Can't write, only read - ok = file:change_mode(Dir, 8#500), - ?assertEqual({error, eacces}, persist(Q1, #{}, #{})), - - Q3 = unpersist(Name), - ?assertEqual(Name, smoosh_priority_queue:name(Q3)), - ?assertEqual(#{max => 0, min => 0, size => 0}, smoosh_priority_queue:info(Q3)), - + % SKIP_TEST_ON_WINDOWS + {Osfamily, _} = os:type(), + case Osfamily of + win32 -> + ok; + _ -> + % Can't write, only read. This works only when not running as root/admin. + ok = file:change_mode(Dir, 8#500), + % Did the setup work? + {ok, #file_info{access = Access}} = file:read_file_info(Dir), + case Access of + read -> + ?assertEqual({error, eacces}, persist(Q1, #{}, #{})), + Q3 = unpersist(Name), + ?assertEqual(Name, smoosh_priority_queue:name(Q3)), + ?assertEqual(#{max => 0, min => 0, size => 0}, smoosh_priority_queue:info(Q3)); + read_write -> + % We must be running as root/admin so we can't really make dir + % unreadable for ourselves so skip this part + ok + end + end, ok = file:del_dir_r(Dir). drain_q(Q) -> diff --git a/test/elixir/lib/couch.ex b/test/elixir/lib/couch.ex index efdfe31eba8..a119095a9f0 100644 --- a/test/elixir/lib/couch.ex +++ b/test/elixir/lib/couch.ex @@ -117,7 +117,7 @@ defmodule Couch do def process_request_body(body) do if is_map(body) do - :jiffy.encode(body) + :jiffy.encode(body, [:use_nil]) else body end @@ -131,7 +131,7 @@ defmodule Couch do content_type = headers[:"Content-Type"] if !!content_type and String.match?(content_type, ~r/application\/json/) do - body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps]) + body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps, :use_nil]) else process_response_body(body) end diff --git a/test/elixir/lib/couch/dbtest.ex b/test/elixir/lib/couch/dbtest.ex index b1ef926ab98..693e6f0f39c 100644 --- a/test/elixir/lib/couch/dbtest.ex +++ b/test/elixir/lib/couch/dbtest.ex @@ -90,7 +90,7 @@ defmodule Couch.DBTest do if prev_value != "" do url = "/_node/#{node}/_config/#{section}/#{key}" headers = ["X-Couch-Persist": "false"] - body = :jiffy.encode(prev_value) + body = :jiffy.encode(prev_value, [:use_nil]) resp = Couch.put(url, headers: headers, body: body) assert resp.status_code == 200 else @@ -109,7 +109,7 @@ defmodule Couch.DBTest do Enum.map(resp.body["all_nodes"], fn node -> url = "/_node/#{node}/_config/#{section}/#{key}" headers = ["X-Couch-Persist": "false"] - body = :jiffy.encode(value) + body = :jiffy.encode(value, [:use_nil]) resp = Couch.put(url, headers: headers, body: body) assert resp.status_code == 200 {node, resp.body} @@ -494,7 +494,7 @@ defmodule Couch.DBTest do Couch.put( "/_node/#{node}/_config/#{setting.section}/#{setting.key}", headers: ["X-Couch-Persist": false], - body: :jiffy.encode(setting.value) + body: :jiffy.encode(setting.value, [:use_nil]) ) assert resp.status_code == 200 @@ -525,7 +525,7 @@ defmodule Couch.DBTest do Couch.put( "/_node/#{node}/_config/#{setting.section}/#{setting.key}", headers: ["X-Couch-Persist": false], - body: :jiffy.encode(value) + body: :jiffy.encode(value, [:use_nil]) ) assert resp.status_code == 200 diff --git a/test/elixir/lib/couch_raw.ex b/test/elixir/lib/couch_raw.ex index 62a0bbd0ed2..641612c9cd7 100644 --- a/test/elixir/lib/couch_raw.ex +++ b/test/elixir/lib/couch_raw.ex @@ -46,7 +46,7 @@ defmodule Rawresp do def process_request_body(body) do if is_map(body) do - :jiffy.encode(body) + :jiffy.encode(body, [:use_nil]) else body end diff --git a/test/elixir/test/all_docs_test.exs b/test/elixir/test/all_docs_test.exs index 3d07e12e89c..63b2ad5e460 100644 --- a/test/elixir/test/all_docs_test.exs +++ b/test/elixir/test/all_docs_test.exs @@ -116,7 +116,7 @@ defmodule AllDocsTest do assert row["key"] == "1" assert row["id"] == "1" assert row["value"]["deleted"] - assert row["doc"] == :null + assert row["doc"] == nil # Add conflicts conflicted_doc1 = %{ diff --git a/test/elixir/test/attachments_multipart_test.exs b/test/elixir/test/attachments_multipart_test.exs index 1161140a396..6a63df0e055 100644 --- a/test/elixir/test/attachments_multipart_test.exs +++ b/test/elixir/test/attachments_multipart_test.exs @@ -152,7 +152,7 @@ defmodule AttachmentMultipartTest do assert Enum.at(sections, 2).headers["Content-Disposition"] == ~s(attachment; filename="bar.txt") - doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["foo.txt"]["follows"] == true assert doc["_attachments"]["bar.txt"]["follows"] == true @@ -175,7 +175,7 @@ defmodule AttachmentMultipartTest do sections = parse_multipart(resp) assert length(sections) == 2 - doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["foo.txt"]["stub"] == true assert doc["_attachments"]["bar.txt"]["follows"] == true @@ -206,7 +206,7 @@ defmodule AttachmentMultipartTest do assert length(inner_sections) == 2 assert Enum.at(inner_sections, 0).headers["Content-Type"] == "application/json" - doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["foo.txt"]["stub"] == true assert doc["_attachments"]["bar.txt"]["follows"] == true @@ -228,7 +228,7 @@ defmodule AttachmentMultipartTest do assert length(sections) == 2 - doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["foo.txt"]["stub"] == true assert doc["_attachments"]["bar.txt"]["follows"] == true assert Enum.at(sections, 1).body == "this is 18 chars l" @@ -377,7 +377,7 @@ defmodule AttachmentMultipartTest do assert length(inner_sections) == 3 assert Enum.at(inner_sections, 0).headers["Content-Type"] == "application/json" - doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["lorem.txt"]["follows"] == true assert doc["_attachments"]["lorem.txt"]["encoding"] == "gzip" assert doc["_attachments"]["data.bin"]["follows"] == true @@ -414,7 +414,7 @@ defmodule AttachmentMultipartTest do # 2 inner sections: a document body section plus 1 attachment data section assert length(inner_sections) == 2 assert Enum.at(inner_sections, 0).headers["Content-Type"] == "application/json" - doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps]) + doc = :jiffy.decode(Enum.at(inner_sections, 0).body, [:return_maps, :use_nil]) assert doc["_attachments"]["lorem.txt"]["follows"] == true assert doc["_attachments"]["lorem.txt"]["encoding"] == "gzip" assert Enum.at(inner_sections, 1).body != lorem @@ -434,7 +434,7 @@ defmodule AttachmentMultipartTest do boundary = Enum.at(String.split(boundary_arg, "="), 1) if String.starts_with?(boundary, ~s(")) do - :jiffy.decode(boundary) + :jiffy.decode(boundary, [:use_nil]) else boundary end diff --git a/test/elixir/test/basics_test.exs b/test/elixir/test/basics_test.exs index 21eb77bcfbf..a76510a13a1 100644 --- a/test/elixir/test/basics_test.exs +++ b/test/elixir/test/basics_test.exs @@ -388,4 +388,19 @@ defmodule BasicsTest do resp = Couch.get("/", headers: ["X-Couch-Request-ID": uuid]) assert resp.headers["X-Couch-Request-ID"] == uuid end + + @tag + test "_all_dbs/_all_docs is not found", _context do + resp = Couch.get("/_all_dbs/_all_docs") + assert resp.status_code == 404 + assert resp.body["error"] == "not_found" + end + + @tag + test "_dbs_info/_all_docs is not found", _context do + resp = Couch.get("/_dbs_info/_all_docs") + assert resp.status_code == 404 + assert resp.body["error"] == "not_found" + end + end diff --git a/test/elixir/test/changes_async_test.exs b/test/elixir/test/changes_async_test.exs index ae8fb41b26c..75362d8a9c8 100644 --- a/test/elixir/test/changes_async_test.exs +++ b/test/elixir/test/changes_async_test.exs @@ -407,7 +407,7 @@ defmodule ChangesAsyncTest do end defp parse_chunk(msg) do - msg.chunk |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps]) + msg.chunk |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps, :use_nil]) end defp parse_event(msg) do @@ -419,7 +419,7 @@ defmodule ChangesAsyncTest do |> Enum.map(fn p -> p |> IO.iodata_to_binary() - |> :jiffy.decode([:return_maps]) + |> :jiffy.decode([:return_maps, :use_nil]) end) end @@ -497,7 +497,7 @@ defmodule ChangesAsyncTest do body_lines |> Enum.filter(fn line -> line != "" end) |> Enum.map(fn line -> - line |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps]) + line |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps, :use_nil]) end) end diff --git a/test/elixir/test/changes_test.exs b/test/elixir/test/changes_test.exs index 8547af59fac..9fff12e7255 100644 --- a/test/elixir/test/changes_test.exs +++ b/test/elixir/test/changes_test.exs @@ -198,7 +198,7 @@ defmodule ChangesTest do assert Enum.member?(changes_ids, "doc1") assert Enum.member?(changes_ids, "doc3") - encoded_doc_ids = doc_ids.doc_ids |> :jiffy.encode() + encoded_doc_ids = doc_ids.doc_ids |> :jiffy.encode([:use_nil]) resp = Couch.get("/#{db_name}/_changes", diff --git a/test/elixir/test/cluster_with_quorum_test.exs b/test/elixir/test/cluster_with_quorum_test.exs index fc3b28a0b69..718e3f41b51 100644 --- a/test/elixir/test/cluster_with_quorum_test.exs +++ b/test/elixir/test/cluster_with_quorum_test.exs @@ -7,14 +7,14 @@ defmodule WithQuorumTest do Test CouchDB API in a cluster without quorum. """ @tag :with_db_name - test "Creating/Deleting DB should return 201-Created/202-Acepted", context do + test "Creating/Deleting DB should return 201-Created/200-OK", context do db_name = context[:db_name] resp = Couch.put("/#{db_name}") msg = "Should return 201-Created" assert resp.status_code in [201, 202], msg resp = Couch.delete("/#{db_name}") - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 200-OK" + assert resp.status_code == 200, msg end @tag :with_db_name @@ -45,7 +45,7 @@ defmodule WithQuorumTest do end @tag :with_db_name - test "Creating-Updating/Deleting doc with overriden quorum should return 202-Acepted/200-OK", + test "Creating-Updating/Deleting doc with overridden quorum should return 202-Accepted/200-OK", context do db_name = context[:db_name] Couch.put("/#{db_name}") @@ -57,7 +57,7 @@ defmodule WithQuorumTest do body: %{:_id => "0", :a => 1} ) - msg = "Should return 202-Acepted" + msg = "Should return 202-Accepted" assert resp.status_code == 202, msg resp = Couch.get("/#{context[:db_name]}/0") @@ -70,7 +70,7 @@ defmodule WithQuorumTest do body: %{:_id => "0", :_rev => rev, :a => 2} ) - msg = "Should return 202-Acepted" + msg = "Should return 202-Accepted" assert resp.status_code == 202, msg resp = Couch.get("/#{context[:db_name]}/0") @@ -114,12 +114,12 @@ defmodule WithQuorumTest do end @tag :with_db_name - test "Bulk docs overriden quorum should return 202-Acepted", context do + test "Bulk docs overridden quorum should return 202-Accepted", context do db_name = context[:db_name] Couch.put("/#{db_name}") docs = create_docs(@doc_range) resp = Couch.post("/#{db_name}/_bulk_docs", query: %{:w => 3}, body: %{docs: docs}) - msg = "Should return 202-Acepted" + msg = "Should return 202-Accepted" assert resp.status_code == 202, msg Couch.delete("/#{db_name}") @@ -152,7 +152,7 @@ defmodule WithQuorumTest do end @tag :with_db_name - test "Attachments overriden quorum should return 202-Acepted", context do + test "Attachments overridden quorum should return 202-Accepted", context do db_name = context[:db_name] Couch.put("/#{db_name}") resp = Couch.post("/#{context[:db_name]}", body: %{:_id => "0"}) @@ -166,7 +166,7 @@ defmodule WithQuorumTest do headers: ["Content-Type": "text/plain;charset=utf-8"] ) - msg = "Should return 202-Acepted" + msg = "Should return 202-Accepted" assert resp.status_code == 202, msg rev = resp.body["rev"] diff --git a/test/elixir/test/cluster_without_quorum_test.exs b/test/elixir/test/cluster_without_quorum_test.exs index e0095c351ce..16341c6ecb3 100644 --- a/test/elixir/test/cluster_without_quorum_test.exs +++ b/test/elixir/test/cluster_without_quorum_test.exs @@ -7,23 +7,24 @@ defmodule WithoutQuorumTest do Test CouchDB API in a cluster without quorum. """ @tag :with_db_name - test "Creating/Deleting DB should return 202-Acepted", context do + test "Creating/Deleting DB should return 201-Created/200-OK", context do db_name = context[:db_name] resp = Couch.put("/#{db_name}") - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg resp = Couch.delete("/#{db_name}") - assert resp.status_code == 202, msg + msg = "Should return 200-OK" + assert resp.status_code == 200, msg end @tag :with_db_name - test "Creating/Updating/Deleting doc should return 202-Acepted", context do + test "Creating/Updating/Deleting doc should return 201-Created/200-OK", context do db_name = context[:db_name] Couch.put("/#{db_name}") resp = Couch.post("/#{context[:db_name]}", body: %{:_id => "0", :a => 1}) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg resp = Couch.get("/#{context[:db_name]}/0") rev = resp.body["_rev"] @@ -31,20 +32,20 @@ defmodule WithoutQuorumTest do resp = Couch.put("/#{context[:db_name]}/0", body: %{:_id => "0", :_rev => rev, :a => 2}) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg resp = Couch.get("/#{context[:db_name]}/0") rev = resp.body["_rev"] resp = Couch.delete("/#{context[:db_name]}/0", query: %{:rev => rev}) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 200-OK" + assert resp.status_code == 200, msg Couch.delete("/#{db_name}") end @tag :with_db_name - test "Creating-Updating/Deleting doc with overriden quorum should return 201-Created/200-OK", + test "Creating-Updating/Deleting doc with overridden quorum should return 201-Created/200-OK", context do db_name = context[:db_name] Couch.put("/#{db_name}") @@ -82,7 +83,7 @@ defmodule WithoutQuorumTest do end @tag :with_db_name - test "Copy doc should return 202-Acepted", context do + test "Copy doc should return 201-Created", context do db_name = context[:db_name] Couch.put("/#{db_name}") @@ -93,27 +94,27 @@ defmodule WithoutQuorumTest do headers = [Destination: "1"] resp = Couch.request(:copy, "/#{context[:db_name]}/0", headers: headers) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg Couch.delete("/#{db_name}") end @doc_range 1..5 @tag :with_db_name - test "Bulk docs should return 202-Acepted", context do + test "Bulk docs should return 201-Created", context do db_name = context[:db_name] Couch.put("/#{db_name}") docs = create_docs(@doc_range) resp = Couch.post("/#{db_name}/_bulk_docs", body: %{docs: docs}) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg Couch.delete("/#{db_name}") end @tag :with_db_name - test "Bulk docs overriden quorum should return 201-Created", context do + test "Bulk docs overridden quorum should return 201-Created", context do db_name = context[:db_name] Couch.put("/#{db_name}") docs = create_docs(@doc_range) @@ -125,7 +126,7 @@ defmodule WithoutQuorumTest do end @tag :with_db_name - test "Attachments should return 202-Acepted", context do + test "Attachments should return 201-Created", context do db_name = context[:db_name] Couch.put("/#{db_name}") resp = Couch.post("/#{context[:db_name]}", body: %{:_id => "0"}) @@ -139,8 +140,8 @@ defmodule WithoutQuorumTest do headers: ["Content-Type": "text/plain;charset=utf-8"] ) - msg = "Should return 202-Acepted" - assert resp.status_code == 202, msg + msg = "Should return 201-Created" + assert resp.status_code == 201, msg rev = resp.body["rev"] resp = Couch.delete("/#{context[:db_name]}/0/foo.txt", query: %{:rev => rev}) @@ -151,7 +152,7 @@ defmodule WithoutQuorumTest do end @tag :with_db_name - test "Attachments overriden quorum should return 201-Created", context do + test "Attachments overridden quorum should return 201-Created", context do db_name = context[:db_name] Couch.put("/#{db_name}") resp = Couch.post("/#{context[:db_name]}", body: %{:_id => "0"}) diff --git a/test/elixir/test/config/search.elixir b/test/elixir/test/config/search.elixir index b49b8d76275..61640ded1dc 100644 --- a/test/elixir/test/config/search.elixir +++ b/test/elixir/test/config/search.elixir @@ -31,6 +31,56 @@ "facet counts, non-empty", "facet counts, empty", "facet ranges, empty", - "facet ranges, non-empty" + "facet ranges, non-empty", + "timeouts do not expose internal state" + ], + "BasicTextTest": [ + "explain options", + "explain with bookmarks" + ], + "ElemMatchTests": [ + "elem match non object" + ], + "LimitTests": [ + "limit field" + ], + "KeyTest": [ + "dot key", + "peso key", + "unicode in fieldname", + "unicode in selector field", + "internal field tests", + "escape period", + "object period" + ], + "TextIndexSelectionTest": [ + "with text", + "no view index", + "with or", + "manual bad text idx" + ], + "MultiTextIndexSelectionTest": [ + "fallback to json with multi text", + "multi text index is error", + "use index works" + ], + "RegexVsTextIndexTest": [ + "regex works with text index" + ], + "NoDefaultFieldTest": [ + "basic", + "other fields exist" + ], + "NoDefaultFieldWithAnalyzer": [ + "basic", + "other fields exist" + ], + "DefaultFieldWithCustomAnalyzer": [ + "basic", + "not analyzed" + ], + "PartitionMangoTest": [ + "explain options (text)", + "explain works with bookmarks (text)" ] } diff --git a/test/elixir/test/config/skip.elixir b/test/elixir/test/config/skip.elixir index be427a7403d..7d39ee2eec8 100644 --- a/test/elixir/test/config/skip.elixir +++ b/test/elixir/test/config/skip.elixir @@ -1,26 +1,7 @@ %{ - "CookieAuthTest": [ - "cookie auth" - ], - "ReaderACLTest": [ - "unrestricted db can be read" - ], "ReplicationTest": [ - "non-admin user on target - remote-to-remote", "non-admin or reader user on source - remote-to-remote", + "non-admin user on target - remote-to-remote", "unauthorized replication cancellation" - ], - "SecurityValidationTest": [ - "Author presence and user security when replicated" - ], - "WithQuorumTest": [ - "Creating/Deleting DB should return 201-Created/202-Acepted" - ], - "WithoutQuorumTest": [ - "Attachments should return 202-Acepted", - "Bulk docs should return 202-Acepted", - "Copy doc should return 202-Acepted", - "Creating/Deleting DB should return 202-Acepted", - "Creating/Updating/Deleting doc should return 202-Acepted" ] } diff --git a/test/elixir/test/config/suite.elixir b/test/elixir/test/config/suite.elixir index 1cbdfccf365..ea8e55bb077 100644 --- a/test/elixir/test/config/suite.elixir +++ b/test/elixir/test/config/suite.elixir @@ -36,7 +36,6 @@ "etags for attachments", "implicit doc creation allows creating docs with a reserved id. COUCHDB-565", "large attachments COUCHDB-366", - "md5 header for attachments", "reads attachment successfully", "saves attachment successfully", "saves binary", @@ -48,7 +47,7 @@ ], "AuthLockoutTest": [ "lockout after multiple failed authentications", - "lockout warning after multiple failed authentications" + "do not lockout after multiple failed authentications" ], "BasicsTest": [ "'+' in document name should encode to '+'", @@ -82,7 +81,9 @@ "_all_docs POST error when multi-get is not a {'key': [...]} structure", "_bulk_docs POST error when body not an object", "oops, the doc id got lost in code nirwana", - "request ID can be specified at the client" + "request ID can be specified at the client", + "_all_dbs/_all_docs is not found", + "_dbs_info/_all_docs is not found" ], "BatchSaveTest": [ "batch post", @@ -156,7 +157,8 @@ "Only JSON strings are accepted" ], "CookieAuthTest": [ - "cookie auth" + "cookie auth", + "header doesn't contain set-cookie" ], "CopyDocTest": [ "Copy doc tests" @@ -257,6 +259,9 @@ "jwt auth with required iss claim", "jwt auth without secret" ], + "LargeDocsTest": [ + "Large docs" + ], "ListViewsTest": [ "COUCHDB-1113", "HTTP header response set after getRow() called in _list function", @@ -358,6 +363,8 @@ "PartitionMangoTest": [ "explain works with non partitioned db", "explain works with partitions", + "explain works with bookmarks", + "explain options", "global query does not use partition index", "global query uses global index", "non-partitioned query using _all_docs and $eq", @@ -413,7 +420,7 @@ "view updates properly remove old keys" ], "PasswordCacheTest": [ - "password cache" + "password hash cache" ], "ProxyAuthTest": [ "proxy auth with secret", @@ -458,8 +465,6 @@ "default headers returned for _scheduler/docs ", "default headers returned for _scheduler/jobs", "filtered replications - remote-to-remote", - "non-admin or reader user on source - remote-to-remote", - "non-admin user on target - remote-to-remote", "replicate with since_seq - remote-to-remote", "replicating attachment without conflict - COUCHDB-885", "replication by doc ids - remote-to-remote", @@ -467,7 +472,6 @@ "replication restarts after filter change - COUCHDB-892 - remote-to-remote", "simple remote-to-remote replication - remote-to-remote", "source database not found with host", - "unauthorized replication cancellation", "validate_doc_update failure replications - remote-to-remote" ], "ReshardAllDocsTest": [ @@ -481,6 +485,9 @@ "split q=2 shards on node1 (2 jobs)", "toggle global state" ], + "ReshardChangesFeedTest": [ + "all_docs after splitting all shards on node1" + ], "RevStemmingTest": [ "revs limit is kept after compaction", "revs limit produces replication conflict ", @@ -493,8 +500,8 @@ "multiple updates with same _rev raise conflict errors" ], "RewriteJSTest": [ - "Test basic js rewrites on test_rewrite_suite_db", - "Test basic js rewrites on test_rewrite_suite_db%2Fwith_slashes", + "basic js rewrites on test_rewrite_suite_db", + "basic js rewrites on test_rewrite_suite_db%2Fwith_slashes", "early response on test_rewrite_suite_db", "early response on test_rewrite_suite_db%2Fwith_slashes", "loop on test_rewrite_suite_db", @@ -505,8 +512,8 @@ "requests with body preserve the query string rewrite on test_rewrite_suite_db%2Fwith_slashes" ], "RewriteTest": [ - "Test basic rewrites on test_rewrite_suite_db", - "Test basic rewrites on test_rewrite_suite_db%2Fwith_slashes", + "basic rewrites on test_rewrite_suite_db", + "basic rewrites on test_rewrite_suite_db%2Fwith_slashes", "loop detection on test_rewrite_suite_db", "loop detection on test_rewrite_suite_db%2Fwith_slashes", "path relative to server on test_rewrite_suite_db", @@ -643,7 +650,8 @@ "GET - invalid parameter combinations get rejected ", "POST - invalid parameter combinations get rejected ", "argument combinations", - "dir works", + "dir ascending works", + "dir descending works", "empty keys", "keys in GET body (group)", "keys in GET parameters", @@ -711,23 +719,160 @@ "view update seq" ], "WithQuorumTest": [ - "Attachments overriden quorum should return 202-Acepted", + "Attachments overridden quorum should return 202-Accepted", "Attachments should return 201-Created", - "Bulk docs overriden quorum should return 202-Acepted", + "Bulk docs overridden quorum should return 202-Accepted", "Bulk docs should return 201-Created", "Copy doc should return 201-Created", "Creating-Updating/Deleting doc should return 201-Created/200-OK", - "Creating-Updating/Deleting doc with overriden quorum should return 202-Acepted/200-OK", - "Creating/Deleting DB should return 201-Created/202-Acepted" + "Creating-Updating/Deleting doc with overridden quorum should return 202-Accepted/200-OK", + "Creating/Deleting DB should return 201-Created/200-OK" ], "WithoutQuorumTest": [ - "Attachments overriden quorum should return 201-Created", - "Attachments should return 202-Acepted", - "Bulk docs overriden quorum should return 201-Created", - "Bulk docs should return 202-Acepted", - "Copy doc should return 202-Acepted", - "Creating-Updating/Deleting doc with overriden quorum should return 201-Created/200-OK", - "Creating/Deleting DB should return 202-Acepted", - "Creating/Updating/Deleting doc should return 202-Acepted" + "Attachments overridden quorum should return 201-Created", + "Attachments should return 201-Created", + "Bulk docs overridden quorum should return 201-Created", + "Bulk docs should return 201-Created", + "Copy doc should return 201-Created", + "Creating-Updating/Deleting doc with overridden quorum should return 201-Created/200-OK", + "Creating/Deleting DB should return 201-Created/200-OK", + "Creating/Updating/Deleting doc should return 201-Created/200-OK" + ], + "BasicFindTest": [ + "simple find", + "bad selector", + "bad limit", + "bad skip", + "bad sort", + "bad fields", + "bad r", + "bad conflicts", + "multi cond and", + "multi cond duplicate field", + "multi cond or", + "multi col idx", + "missing not indexed", + "limit", + "skip", + "sort", + "sort desc complex", + "sort with primary sort not in selector", + "sort exists true", + "sort desc complex error", + "fields", + "r", + "empty", + "empty subsel", + "empty subsel match", + "unsatisfiable range", + "explain view args", + "explain options", + "explain with bookmarks", + "sort with all docs" + ], + "IgnoreDesignDocsForAllDocsIndexTests": [ + "should not return design docs" + ], + "ChooseCorrectIndexForDocs": [ + "choose index with one field in index", + "choose index with two", + "choose index alphabetically", + "choose index most accurate", + "choose index most accurate in memory selector", + "warn on full db scan", + "chooses idxA", + "can query with range on secondary column", + "choose index with id", + "choose index with rev" + ], + "RegularCoveringIndexTest": [ + "index covers query 1 field index id", + "index covers query 2 field index id", + "index covers query 2 field index extract field", + "index covers query 2 field index extract field force index", + "index covers query elemMatch", + "index covers query composite field_id", + "index does not cover query empty selector", + "index does not cover query field not in index", + "index does not cover query all fields", + "index does not cover query partial selector id", + "index does not cover query partial selector", + "index does not cover selector with more fields", + "covering index provides correct answer 2 field index", + "covering index provides correct answer id" + ], + "PartitionedCoveringIndexTest": [ + "index covers query 1 field index id", + "index covers query 2 field index id", + "index covers query 2 field index extract field", + "index covers query 2 field index extract field force index", + "index covers query elemMatch", + "index covers query composite field_id", + "index does not cover query empty selector", + "index does not cover query field not in index", + "index does not cover query all fields", + "index does not cover query partial selector id", + "index does not cover query partial selector", + "index does not cover selector with more fields", + "covering index provides correct answer 2 field index", + "covering index provides correct answer id" + ], + "BeginsWithOperator": [ + "basic", + "json range", + "compound key", + "sort", + "all docs range", + "no index", + "invalid operand", + "does not match non string value", + "no matches", + "case sensitivity" + ], + "JSONIndexSelectionTest": [ + "basic", + "with and", + "with nested and", + "with or", + "use most columns", + "no valid sort index", + "invalid use index", + "uses index when no range or equals", + "reject use index invalid fields", + "reject use index ddoc and name invalid fields", + "reject use index sort order", + "use index fallback if valid sort", + "prefer use index over optimal index", + "manual bad view idx01", + "explain sort reverse", + "use index with invalid name", + "use index without fallback succeeds for valid index", + "use index without fallback fails for invalid index with fallback available", + "use index without fallback succeeds for empty index", + "use index without fallback fails for empty index", + "use index without fallback fails for invalid index no fallback exists", + "index without fallback", + "no index without fallback", + "uses all docs when fields do not match selector", + "uses all docs when selector doesnt require fields to exist" + ], + "JwtRolesClaimTest": [ + "case: roles_claim_name (defined) / roles_claim_path (defined)", + "case: roles_claim_name (defined) / roles_claim_path (undefined)", + "case: roles_claim_name (undefined) / roles_claim_path (defined)", + "case: roles_claim_name (undefined) / roles_claim_path (undefined)", + "case: roles_claim_path with bad input" + ], + "PaginateJsonDocs": [ + "all docs paginate to end", + "return previous bookmark for empty", + "all docs with skip", + "all docs reverse", + "bad bookmark", + "throws error on text bookmark", + "index pagination", + "index pagination two keys", + "index pagination reverse", + "index pagination same emitted key" ] } diff --git a/test/elixir/test/config_test.exs b/test/elixir/test/config_test.exs index 5a5140840bf..d1a4f7d9c21 100644 --- a/test/elixir/test/config_test.exs +++ b/test/elixir/test/config_test.exs @@ -23,7 +23,7 @@ defmodule ConfigTest do def set_config(context, section, key, val, status_assert) do url = "#{context[:config_url]}/#{section}/#{key}" headers = ["X-Couch-Persist": "false"] - resp = Couch.put(url, headers: headers, body: :jiffy.encode(val)) + resp = Couch.put(url, headers: headers, body: :jiffy.encode(val, [:use_nil])) if status_assert do assert resp.status_code == status_assert diff --git a/test/elixir/test/cookie_auth_test.exs b/test/elixir/test/cookie_auth_test.exs index cf9b36c7f3a..7cbac699fc1 100644 --- a/test/elixir/test/cookie_auth_test.exs +++ b/test/elixir/test/cookie_auth_test.exs @@ -374,45 +374,14 @@ defmodule CookieAuthTest do login("jan", "apple") end - test "basic+cookie auth interaction" do + test "header doesn't contain set-cookie" do # performing a successful basic authentication will create a session cookie resp = Couch.get( - "/_all_dbs", - no_auth: true, - headers: [authorization: "Basic #{:base64.encode("jan:apple")}"]) - assert resp.status_code == 200 - - # extract cookie value - cookie = resp.headers[:"set-cookie"] - [token | _] = String.split(cookie, ";") - - # Cookie is usable on its own - resp = Couch.get( - "/_session", - no_auth: true, - headers: [cookie: token]) - assert resp.status_code == 200 - assert resp.body["userCtx"]["name"] == "jan" - assert resp.body["info"]["authenticated"] == "cookie" + "/_all_dbs", + no_auth: true, + headers: [authorization: "Basic #{:base64.encode("jan:apple")}"]) - # Cookie is usable with basic auth if usernames match - resp = Couch.get( - "/_session", - no_auth: true, - headers: [ - authorization: "Basic #{:base64.encode("jan:apple")}", - cookie: token]) assert resp.status_code == 200 - assert resp.body["userCtx"]["name"] == "jan" - assert resp.body["info"]["authenticated"] == "cookie" - - # Cookie is not usable with basic auth if usernames don't match - resp = Couch.get( - "/_session", - no_auth: true, - headers: [ - authorization: "Basic #{:base64.encode("notjan:banana")}", - cookie: token]) - assert resp.status_code == 401 + assert resp.headers["set-cookie"] == nil end end diff --git a/test/elixir/test/design_docs_test.exs b/test/elixir/test/design_docs_test.exs index 46f12e1532a..8ef744bdfb4 100644 --- a/test/elixir/test/design_docs_test.exs +++ b/test/elixir/test/design_docs_test.exs @@ -222,7 +222,7 @@ defmodule DesignDocsTest do result = resp.body |> IO.iodata_to_binary() - |> :jiffy.decode([:return_maps]) + |> :jiffy.decode([:return_maps, :use_nil]) assert result["language"] == "javascript" end diff --git a/test/elixir/test/design_options_test.exs b/test/elixir/test/design_options_test.exs index 95a938e380c..02015b55da6 100644 --- a/test/elixir/test/design_options_test.exs +++ b/test/elixir/test/design_options_test.exs @@ -50,7 +50,7 @@ defmodule DesignOptionsTest do row_with_key = resp.body["rows"] - |> Enum.filter(fn p -> p["key"] != :null end) + |> Enum.filter(fn p -> p["key"] != nil end) assert length(row_with_key) == 2 end diff --git a/test/elixir/test/disk_monitor.exs b/test/elixir/test/disk_monitor_test.exs similarity index 100% rename from test/elixir/test/disk_monitor.exs rename to test/elixir/test/disk_monitor_test.exs diff --git a/test/elixir/test/erlang_views_test.exs b/test/elixir/test/erlang_views_test.exs index 3346c22748f..ceacfeb3d31 100644 --- a/test/elixir/test/erlang_views_test.exs +++ b/test/elixir/test/erlang_views_test.exs @@ -96,7 +96,7 @@ defmodule ErlangViewsTest do "erlang" ) - assert Map.get(List.first(results["rows"]), "key", :null) == :null + assert Map.get(List.first(results["rows"]), "key", nil) == nil assert List.first(results["rows"])["value"] > 0 end diff --git a/test/elixir/test/jsonp_test.exs b/test/elixir/test/jsonp_test.exs index 169f663879b..0dbb59b781d 100644 --- a/test/elixir/test/jsonp_test.exs +++ b/test/elixir/test/jsonp_test.exs @@ -107,7 +107,7 @@ defmodule JsonpTest do |> Enum.map(fn p -> p |> IO.iodata_to_binary() - |> :jiffy.decode([:return_maps]) + |> :jiffy.decode([:return_maps, :use_nil]) end) |> Enum.at(0) diff --git a/test/elixir/test/jwt_roles_claim_test.exs b/test/elixir/test/jwt_roles_claim_test.exs index 28b280e9ca4..ec3c7e53e0d 100644 --- a/test/elixir/test/jwt_roles_claim_test.exs +++ b/test/elixir/test/jwt_roles_claim_test.exs @@ -14,15 +14,15 @@ defmodule JwtRolesClaimTest do %{ :section => "jwt_keys", :key => "hmac:myjwttestkey", + # "a-string-secret-at-least-256-bits-long" :value => ~w( - NTNv7j0TuYARvmNMmWXo6fKvM4o6nv/aUi9ryX38ZH+L1bkrnD1ObOQ8JAUmHCBq7 - Iy7otZcyAagBLHVKvvYaIpmMuxmARQ97jUVG16Jkpkp1wXOPsrF9zwew6TpczyH - kHgX5EuLg2MeBuiT/qJACs1J0apruOOJCg/gOtkjB4c= + YS1zdHJpbmctc2VjcmV0LWF0LWxlYXN0LTI1Ni1iaXRzLWxvbmc= ) |> Enum.join() }, %{ :section => "jwt_keys", :key => "hmac:myjwttestkey2", + # "Undoubtedly-Engaging-Roadway-029" :value => ~w( VW5kb3VidGVkbHktRW5nYWdpbmctUm9hZHdheS0wMjk= ) |> Enum.join() @@ -133,17 +133,19 @@ defmodule JwtRolesClaimTest do end def test_roles(roles) do + # myjwttestkey + # Token expires after Thu Dec 31 2099 22:59:59 GMT+0000 token = ~w( eyJ0eXAiOiJKV1QiLCJraWQiOiJteWp3dHRlc3RrZXkiLCJhbGciOiJIUzI1NiJ9. eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRyd - WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxNzU1Mjk5NDEwLCJteSI6eyJuZXN0ZW + WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxOTI0OTg4Mzk5LCJteSI6eyJuZXN0ZW QiOnsiX2NvdWNoZGIucm9sZXMiOlsibXlfbmVzdGVkX2NvdWNoZGIucm9sZXNfMSI sIm15X25lc3RlZF9jb3VjaGRiLnJvbGVzXzEiXX19LCJfY291Y2hkYi5yb2xlcyI6 WyJfY291Y2hkYi5yb2xlc18xIiwiX2NvdWNoZGIucm9sZXNfMiJdLCJteS5fY291Y 2hkYi5yb2xlcyI6WyJteS5fY291Y2hkYi5yb2xlc18xIiwibXkuX2NvdWNoZGIucm 9sZXNfMiJdLCJmb28iOnsiYmFyLnpvbmsiOnsiYmF6LmJ1dSI6eyJiYWEiOnsiYmF hLmJlZSI6eyJyb2xlcyI6WyJteV9uZXN0ZWRfcm9sZV8xIiwibXlfbmVzdGVkX3Jv - bGVfMiJdfX19fX19.F6kQK-FK0z1kP01bTyw-moXfy2klWfubgF7x7Xitd-0) |> Enum.join() + bGVfMiJdfX19fX19._3BgqHB8ETk4QPOn_LMgUkjgsjgqDCf7AUlV3XRmP6A) |> Enum.join() resp = Couch.get("/_session", @@ -156,19 +158,20 @@ defmodule JwtRolesClaimTest do end def test_roles_as_string(roles) do - # Different token + # myjwttestkey2 + # Token expires after Thu Dec 31 2099 22:59:59 GMT+0000 token = ~w( eyJ0eXAiOiJKV1QiLCJraWQiOiJteWp3dHRlc3RrZXkyIiwiYWxnIjoiSFMyNTYifQ. eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWU - sImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxNzU1Mjk5NDEwLCJteSI6eyJuZXN0ZWQiOn + sImlhdCI6MTY1NTI5NTgxMCwiZXhwIjo0MTAyNDQxMTk5LCJteSI6eyJuZXN0ZWQiOn siX2NvdWNoZGIucm9sZXMiOiJteV9uZXN0ZWRfY291Y2hkYl9zdHJpbmcucm9sZXNfM SwgbXlfbmVzdGVkX2NvdWNoZGJfc3RyaW5nLnJvbGVzXzEifX0sIl9jb3VjaGRiLnJv bGVzIjoiX2NvdWNoZGJfc3RyaW5nLnJvbGVzXzEsX2NvdWNoZGJfc3RyaW5nLnJvbGV zXzIiLCJteS5fY291Y2hkYi5yb2xlcyI6Im15Ll9jb3VjaGRiX3N0cmluZy5yb2xlc1 8xLCBteS5fY291Y2hkYl9zdHJpbmcucm9sZXNfMiIsImZvbyI6eyJiYXIuem9uayI6e yJiYXouYnV1Ijp7ImJhYSI6eyJiYWEuYmVlIjp7InJvbGVzIjoibXlfbmVzdGVkX3N0 - cmluZ19yb2xlXzEsIG15X25lc3RlZF9zdHJpbmdfcm9sZV8yIn19fX19fQ.rzaLmcA2 - 0R291XuGYNNTM9ypGL3UD_GlVp3DmBtWrZI + cmluZ19yb2xlXzEsIG15X25lc3RlZF9zdHJpbmdfcm9sZV8yIn19fX19fQ.nUQYe_Fy + 1LBY0F4jbfLwj47p2468v0lrCzHXMWpkfA4 ) |> Enum.join() resp = @@ -182,17 +185,19 @@ defmodule JwtRolesClaimTest do end def test_roles_with_bad_input() do + # myjwttestkey + # Token expires after Thu Dec 31 2099 22:59:59 GMT+0000 token = ~w( eyJ0eXAiOiJKV1QiLCJraWQiOiJteWp3dHRlc3RrZXkiLCJhbGciOiJIUzI1NiJ9. eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRyd - WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxNzU1Mjk5NDEwLCJteSI6eyJuZXN0ZW + WUsImlhdCI6MTY1NTI5NTgxMCwiZXhwIjoxOTI0OTg4Mzk5LCJteSI6eyJuZXN0ZW QiOnsiX2NvdWNoZGIucm9sZXMiOlsibXlfbmVzdGVkX2NvdWNoZGIucm9sZXNfMSI sIm15X25lc3RlZF9jb3VjaGRiLnJvbGVzXzEiXX19LCJfY291Y2hkYi5yb2xlcyI6 WyJfY291Y2hkYi5yb2xlc18xIiwiX2NvdWNoZGIucm9sZXNfMiJdLCJteS5fY291Y 2hkYi5yb2xlcyI6WyJteS5fY291Y2hkYi5yb2xlc18xIiwibXkuX2NvdWNoZGIucm 9sZXNfMiJdLCJmb28iOnsiYmFyLnpvbmsiOnsiYmF6LmJ1dSI6eyJiYWEiOnsiYmF hLmJlZSI6eyJyb2xlcyI6WyJteV9uZXN0ZWRfcm9sZV8xIiwibXlfbmVzdGVkX3Jv - bGVfMiJdfX19fX19.F6kQK-FK0z1kP01bTyw-moXfy2klWfubgF7x7Xitd-0) |> Enum.join() + bGVfMiJdfX19fX19._3BgqHB8ETk4QPOn_LMgUkjgsjgqDCf7AUlV3XRmP6A) |> Enum.join() resp = Couch.get("/_session", diff --git a/test/elixir/test/large_docs_text.exs b/test/elixir/test/large_docs_test.exs similarity index 100% rename from test/elixir/test/large_docs_text.exs rename to test/elixir/test/large_docs_test.exs diff --git a/test/elixir/test/mango/02_basic_find_test.exs b/test/elixir/test/mango/02_basic_find_test.exs new file mode 100644 index 00000000000..0285d8d5336 --- /dev/null +++ b/test/elixir/test/mango/02_basic_find_test.exs @@ -0,0 +1,337 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule BasicFindTest do + use CouchTestCase + + @db_name "basic-find" + + setup do + UserDocs.setup(@db_name) + end + + test "bad selector" do + bad_selectors = [ + nil, + true, + false, + 1.0, + "foobarbaz", + %{"foo" => %{"$not_an_op" => 2}}, + %{"$gt" => 2}, + [nil, "bing"], + %{"_id" => %{"" => nil}}, + ] + Enum.each(bad_selectors, fn bs -> + {:error, resp} = MangoDatabase.find(@db_name, bs) + assert resp.status_code == 400 + end) + end + + test "bad limit" do + bad_limits = [nil, true, false, -1, 1.2, "no limit!", %{"foo" => "bar"}, [2]] + Enum.each(bad_limits, fn bl -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, limit: bl, sort: [%{"age" => "asc"}]) + assert resp.status_code == 400 + end) + end + + test "bad skip" do + bad_skips = [nil, true, false, -3, 1.2, "no limit!", %{"foo" => "bar"}, [2]] + Enum.each(bad_skips, fn bs -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, skip: bs, sort: [%{"age" => "asc"}]) + assert resp.status_code == 400 + end) + end + + test "bad sort" do + bad_sorts = [ + nil, + true, + false, + 1.2, + "no limit!", + %{"foo" => "bar"}, + [2], + [%{"foo" => "asc", "bar" => "asc"}], + [%{"foo" => "asc"}, %{"bar" => "desc"}] + ] + Enum.each(bad_sorts, fn bs -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, sort: bs) + assert resp.status_code == 400 + end) + end + + test "bad fields" do + bad_fields = [ + nil, + true, + false, + 1.2, + "no limit!", + %{"foo" => "bar"}, + [2], + [[]], + ["foo", 2.0], + ] + Enum.each(bad_fields, fn bf -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, fields: bf) + assert resp.status_code == 400 + end) + end + + test "bad r" do + bad_rs = [nil, true, false, 1.2, "no limit!", %{"foo" => "bar"}, [2]] + Enum.each(bad_rs, fn br -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, r: br) + assert resp.status_code == 400 + end) + end + + test "bad conflicts" do + bad_conflicts = [nil, 1.2, "no limit!", %{"foo" => "bar"}, [2]] + Enum.each(bad_conflicts, fn bc -> + {:error, resp} = MangoDatabase.find(@db_name, %{"int" => %{"$gt" => 2}}, conflicts: bc) + assert resp.status_code == 400 + end) + end + + test "simple find" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$lt" => 35}}) + user_ids = Enum.map(docs, fn doc -> doc["user_id"] end) + + assert user_ids == [9, 1, 7] + end + + test "multi cond and" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"manager" => true, "location.city" => "Longbranch"}) + + user_id = Enum.map(docs, fn doc -> doc["user_id"] end) + assert user_id == [7] + end + + test "multi cond duplicate field" do + # need to explicitly define JSON as dict won't allow duplicate keys + body = ~s({ + "selector": { + "location.city": {"$regex": "^L+"}, + "location.city": {"$exists": true} + } + }) + resp = Couch.post("/#{@db_name}/_find", body: body) + # expectation is that only the second instance + # of the "location.city" field is used + assert length(resp.body["docs"]) == 15 + end + + test "multi cond or" do + {:ok, docs} = MangoDatabase.find( + @db_name, + %{ + "$and" => [ + %{"age" => %{"$gte" => 75}}, + %{"$or" => [%{"name.first" => "Mathis"}, %{"name.first" => "Whitley"}]}, + ] + } + ) + + user_id = Enum.map(docs, fn doc -> doc["user_id"] end) + assert user_id == [11, 13] + end + + test "multi col idx" do + {:ok, docs} = MangoDatabase.find( + @db_name, + %{ + "location.state" => %{"$and" => [%{"$gt" => "Hawaii"}, %{"$lt" => "Maine"}]}, + "location.city" => %{"$lt" => "Longbranch"}, + } + ) + + user_id = Enum.map(docs, fn doc -> doc["user_id"] end) + assert user_id == [6] + end + + test "missing not indexed" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"favorites.3" => "C"}) + user_id = Enum.map(docs, fn doc -> doc["user_id"] end) + assert user_id == [6] + + {:ok, docs} = MangoDatabase.find(@db_name, %{"favorites.3" => nil}) + assert Enum.empty?(docs) + + {:ok, docs} = MangoDatabase.find(@db_name, %{"twitter" => %{"$gt" => nil}}) + user_id = Enum.map(docs, fn doc -> doc["user_id"] end) + assert user_id == [1, 4, 0, 13] + end + + test "limit" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}) + assert length(docs) == 15 + + Enum.each([0, 1, 5, 14], fn l -> + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, limit: l, sort: [%{"age" => "asc"}]) + assert length(docs) == l + end) + end + + test "skip" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}) + assert length(docs) == 15 + + Enum.each([0, 1, 5, 14], fn s -> + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, skip: s, sort: [%{"age" => "asc"}]) + assert length(docs) == (15 - s) + end) + end + + test "sort" do + {:ok, docs1} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, sort: [%{"age" => "asc"}]) + docs2 = Enum.sort_by(docs1, fn d -> d["age"] end) + assert docs1 == docs2 + + {:ok, docs1} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, sort: [%{"age" => "desc"}]) + docs2 = Enum.reverse(Enum.sort_by(docs1, fn d -> d["age"] end)) + assert docs1 == docs2 + end + + test "sort desc complex" do + {:ok, docs} = MangoDatabase.find(@db_name, + %{ + "company" => %{"$lt" => "M"}, + "$or" => [%{"company" => "Dreamia"}, %{"manager" => true}], + }, + sort: [%{"company" => "desc"}, %{"manager" => "desc"}] + ) + companies_returned = Enum.map(docs, fn doc -> doc["company"] end) + desc_companies = Enum.sort(companies_returned, :desc) + assert desc_companies == companies_returned + end + + test "sort with primary sort not in selector" do + {:error, resp} = MangoDatabase.find(@db_name, + %{"name.last" => %{"$lt" => "M"}}, + sort: [%{"name.first" => "desc"}] + ) + assert resp.status_code == 400 + assert resp.body["error"] == "no_usable_index" + end + + test "sort exists true" do + {:ok, docs1} = MangoDatabase.find(@db_name, + %{"age" => %{"$gt" => 0, "$exists" => true}}, + sort: [%{"age" => "asc"}] + ) + docs2 = Enum.sort_by(docs1, fn d -> d["age"] end) + assert docs1 == docs2 + end + + test "sort desc complex error" do + {:error, resp} = MangoDatabase.find(@db_name, + %{ + "company" => %{"$lt" => "M"}, + "$or": [%{"company" => "Dreamia"}, %{"manager" => true}], + }, + sort: [%{"company" => "desc"}] + ) + assert resp.status_code == 400 + assert resp.body["error"] == "no_usable_index" + end + + test "fields" do + selector = %{"age" => %{"$gt" => 0}} + {:ok, docs} = MangoDatabase.find(@db_name, selector, fields: ["user_id", "location.address"]) + Enum.each(docs, fn d -> + assert Enum.sort(Map.keys(d)) == ["location", "user_id"] + assert Enum.sort(Map.keys(d["location"])) == ["address"] + end) + end + + test "r" do + Enum.each([1, 2, 3], fn r -> + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, r: r) + assert length(docs) == 15 + end) + end + + test "empty" do + {:ok, docs} = MangoDatabase.find(@db_name, %{}) + # 15 users + assert length(docs) == 15 + end + + test "empty subsel" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"_id" => %{"$gt" => nil}, "location" => %{}}) + assert Enum.empty?(docs) + end + + test "empty subsel match" do + _resp = MangoDatabase.save_docs(@db_name, [%{"user_id" => "eo", "empty_obj" => %{}}]) + {:ok, docs} = MangoDatabase.find(@db_name, %{"_id" => %{"$gt" => nil}, "empty_obj" => %{}}) + assert length(docs) == 1 + assert Enum.at(docs, 0)["user_id"] == "eo" + end + + test "unsatisfiable range" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"$and" => [%{"age" => %{"$gt" => 0}}, %{"age" => %{"$lt" => 0}}]}) + assert Enum.empty?(docs) + end + + test "explain view args" do + {:ok, explain} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, fields: ["manager"], explain: true) + assert explain["mrargs"]["stable"] == false + assert explain["mrargs"]["update"] == true + assert explain["mrargs"]["reduce"] == false + assert explain["mrargs"]["start_key"] == [0] + assert explain["mrargs"]["end_key"] == [""] + assert explain["mrargs"]["include_docs"] == true + end + + test "explain options" do + {:ok, explain} = MangoDatabase.find(@db_name, %{"age" => %{"$gt" => 0}}, fields: ["manager"], explain: true) + opts = explain["opts"] + assert opts["r"] == 1 + assert opts["limit"] == 25 + assert opts["skip"] == 0 + assert opts["fields"] == ["manager"] + assert opts["sort"] == %{} + assert opts["bookmark"] == "nil" + assert opts["conflicts"] == false + assert opts["execution_stats"] == false + assert opts["partition"] == "" + assert opts["stable"] == false + assert opts["stale"] == false + assert opts["update"] == true + assert opts["use_index"] == [] + assert opts["allow_fallback"] == true + end + + test "explain with bookmarks" do + query = %{"age" => %{"$gt" => 42}} + + {:ok, resp} = MangoDatabase.find(@db_name, query, limit: 1, return_raw: true) + assert length(resp["docs"]) == 1 + assert resp["bookmark"] != "nil" + {:ok, explain} = MangoDatabase.find(@db_name, query, bookmark: resp["bookmark"], explain: true) + assert is_binary(explain["opts"]["bookmark"]) + assert resp["bookmark"] == explain["opts"]["bookmark"] + end + + test "sort with all docs" do + {:ok, explain} = MangoDatabase.find(@db_name, + %{"_id" => %{"$gt" => 0}, "age" => %{"$gt" => 0}}, + sort: ["_id"], explain: true + ) + assert explain["index"]["type"] == "special" + end + +end diff --git a/test/elixir/test/mango/04_key_test.exs b/test/elixir/test/mango/04_key_test.exs new file mode 100644 index 00000000000..c54813bddbe --- /dev/null +++ b/test/elixir/test/mango/04_key_test.exs @@ -0,0 +1,177 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule KeyTest do + use CouchTestCase + + @db_name "key-test" + + @test_docs [ + %{"_id" => "100", "type" => "complex_key", "title" => "normal key"}, + %{ + "_id" => "200", + "type" => "complex_key", + "title" => "key with dot", + "dot.key" => "dot's value", + "none" => %{"dot" => "none dot's value"}, + "name.first" => "Kvothe", + }, + %{ + "_id" => "300", + "type" => "complex_key", + "title" => "key with peso", + "$key" => "peso", + "deep" => %{"$key" => "deep peso"}, + "name" => %{"first" => "Master Elodin"}, + }, + %{"_id" => "400", "type" => "complex_key", "title" => "unicode key", "" => "apple"}, + %{ + "_id" => "500", + "title" => "internal_fields_format", + "utf8-1[]:string" => "string", + "utf8-2[]:boolean[]" => true, + "utf8-3[]:number" => 9, + "utf8-3[]:null" => nil, + }, + ] + + setup do + MangoDatabase.recreate(@db_name) + MangoDatabase.save_docs(@db_name, @test_docs, w: 3) + MangoDatabase.create_index(@db_name, ["type"], ddoc: "view") + MangoDatabase.create_text_index(@db_name, ddoc: "text") + :ok + end + + defp run_check(query, check, opts) do + fields = Keyword.get(opts, :fields, []) + indexes = Keyword.get(opts, :indexes, ["view", "text"]) + Enum.each(indexes, fn idx -> + {:ok, docs} = MangoDatabase.find(@db_name, query, fields: fields, use_index: idx) + check.(docs) + end) + end + + test "dot key" do + query = %{"type" => "complex_key"} + fields = ["title", "dot\\.key", "none.dot"] + + check = fn docs -> + assert length(docs) == 4 + doc = Enum.at(docs, 1) + assert Map.has_key?(doc, "dot.key") + assert doc["dot.key"] == "dot's value" + assert Map.has_key?(doc, "none") + assert doc["none"]["dot"] == "none dot's value" + end + + run_check(query, check, fields: fields) + end + + test "peso key" do + query = %{"type" => "complex_key"} + fields = ["title", "$key", "deep.$key"] + + check = fn docs -> + assert length(docs) == 4 + doc = Enum.at(docs, 2) + assert Map.has_key?(doc, "$key") + assert doc["$key"] == "peso" + assert Map.has_key?(doc, "deep") + assert doc["deep"]["$key"] == "deep peso" + end + + run_check(query, check, fields: fields) + end + + test "unicode in fieldname" do + query = %{"type" => "complex_key"} + fields = ["title", ""] + + check = fn docs -> + assert length(docs) == 4 + doc = Enum.at(docs, 3) + # note:  == \uf8ff + assert Map.has_key?(doc, "\uf8ff" ) + assert doc["\uf8ff"] == "apple" + end + + run_check(query, check, fields: fields) + end + + # The rest of these tests are only run against the text + # indexes because view indexes don't have to worry about + # field *name* escaping in the index. + + test "unicode in selector field" do + query = %{"" => "apple"} + + check = fn docs -> + assert length(docs) == 1 + doc = Enum.at(docs, 0) + assert doc["\uf8ff"] == "apple" + end + + run_check(query, check, indexes: ["text"]) + end + + test "internal field tests" do + queries = [ + %{"utf8-1[]:string" => "string"}, + %{"utf8-2[]:boolean[]" => true}, + %{"utf8-3[]:number" => 9}, + %{"utf8-3[]:null" => nil}, + ] + + check = fn docs -> + assert length(docs) == 1 + doc = Enum.at(docs, 0) + assert doc["title"] == "internal_fields_format" + end + + Enum.each(queries, fn query -> + run_check(query, check, indexes: ["text"]) + end) + end + + test "escape period" do + query = %{"name\\.first" => "Kvothe"} + check = fn docs -> + assert length(docs) == 1 + doc = Enum.at(docs, 0) + assert doc["name.first"] == "Kvothe" + end + run_check(query, check, indexes: ["text"]) + + query = %{"name.first" => "Kvothe"} + check_empty = fn docs -> + assert Enum.empty?(docs) + end + run_check(query, check_empty, indexes: ["text"]) + end + + test "object period" do + query = %{"name.first" => "Master Elodin"} + check = fn docs -> + assert length(docs) == 1 + doc = Enum.at(docs, 0) + assert doc["title"] == "key with peso" + end + run_check(query, check, indexes: ["text"]) + + query = %{"name\\.first" => "Master Elodin"} + check_empty = fn docs -> + assert Enum.empty?(docs) + end + run_check(query, check_empty, indexes: ["text"]) + end +end diff --git a/test/elixir/test/mango/05_index_selection_test.exs b/test/elixir/test/mango/05_index_selection_test.exs new file mode 100644 index 00000000000..92358e40b1b --- /dev/null +++ b/test/elixir/test/mango/05_index_selection_test.exs @@ -0,0 +1,440 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule IndexSelectionTest do + use CouchTestCase + defmacro describe(db) do + quote do + test "basic" do + {:ok, resp} = MangoDatabase.find(unquote(db), %{"age" => 123}, explain: true) + assert resp["index"]["type"] == "json" + end + + test "with and" do + {:ok, resp} = MangoDatabase.find(unquote(db), + %{ + "name.first" => "Stephanie", + "name.last" => "This doesn't have to match anything.", + }, + explain: true + ) + assert resp["index"]["type"] == "json" + end + + test "with nested and" do + {:ok, resp} = MangoDatabase.find(unquote(db), + %{ + "name.first" => %{"$gt" => "a", "$lt" => "z"}, + "name.last" => "Foo" + }, + explain: true + ) + assert resp["index"]["type"] == "json" + end + + test "with or" do + ddocid = "_design/company_and_manager" + {:ok, resp} = MangoDatabase.find(unquote(db), + %{ + "company" => %{"$gt" => "a", "$lt" => "z"}, + "$or" => [%{"manager" => "Foo"}, %{"manager" => "Bar"}], + }, + explain: true + ) + assert resp["index"]["ddoc"] == ddocid + end + + test "use most columns" do + ddocid = "_design/age" + {:ok, resp} = MangoDatabase.find(unquote(db), + %{ + "name.first" =>"Stephanie", + "name.last" => "Something or other", + "age" => %{"$gt" => 1}, + }, + explain: true + ) + assert resp["index"]["ddoc"] != ddocid + + {:ok, resp} = MangoDatabase.find(unquote(db), + %{ + "name.first" => "Stephanie", + "name.last" => "Something or other", + "age" => %{"$gt" => 1}, + }, + use_index: ddocid, + explain: true + ) + assert resp["index"]["ddoc"] == ddocid + end + + test "no valid sort index" do + {:error, resp} = MangoDatabase.find(unquote(db), + %{ + "_id" => %{"$gt" => nil} + }, + sort: ["name"], + return_raw: true + ) + assert resp.status_code == 400 + end + + test "invalid use index" do + # ddoc id for the age index + ddocid = "_design/age" + {:ok, r} = MangoDatabase.find(unquote(db), %{}, use_index: ddocid, return_raw: true) + result = + r["warning"] + |> String.split("\n") + |> Enum.at(0) + |> String.downcase() + expected = "#{ddocid} was not used because it does not contain a valid index for this query." + assert result == expected + end + + test "uses index when no range or equals" do + # index on ["manager"] should be valid because + # selector requires "manager" to exist. The + # selector doesn't narrow the keyrange so it's + # a full index scan + selector = %{"manager" => %{"$exists" => true}} + {:ok, docs} = MangoDatabase.find(unquote(db), selector) + assert length(docs) == 14 + + {:ok, resp_explain} = MangoDatabase.find(unquote(db), selector, explain: true) + assert resp_explain["index"]["type"] == "json" + end + + test "reject use index invalid fields" do + ddocid = "_design/company_and_manager" + selector = %{"company" => "Pharmex"} + {:ok, r} = MangoDatabase.find(unquote(db), selector, use_index: ddocid, return_raw: true) + result = + r["warning"] + |> String.split("\n") + |> Enum.at(0) + |> String.downcase() + expected = "#{ddocid} was not used because it does not contain a valid index for this query." + assert result == expected + + # should still return a correct result + assert Enum.empty?(r["docs"]) == false + Enum.each(r["docs"], fn d -> + assert d["company"] == "Pharmex" + end) + end + + test "reject use index ddoc and name invalid fields" do + ddocid = "_design/company_and_manager" + name = "company_and_manager" + selector = %{"company" => "Pharmex"} + + {:ok, resp} = MangoDatabase.find(unquote(db), selector, use_index: [ddocid, name], return_raw: true) + result = + resp["warning"] + |> String.split("\n") + |> Enum.at(0) + |> String.downcase() + expected = "#{ddocid}, #{name} was not used because it is not a valid index for this query." + assert result == expected + + # should still return a correct result + Enum.each(resp["docs"], fn d -> + assert d["company"] == "Pharmex" + end) + end + + test "reject use index sort order" do + # index on ["company","manager"] which should not be valid + # and there is no valid fallback (i.e. an index on ["company"]) + ddocid = "_design/company_and_manager" + selector = %{"company" => %{"$gt" => nil}} + {:error, resp} = MangoDatabase.find(unquote(db), selector, use_index: ddocid, sort: [%{"company" => "desc"}]) + assert resp.status_code == 400 + end + + test "use index fallback if valid sort" do + ddocid_valid = "_design/fallbackfoo" + ddocid_invalid = "_design/fallbackfoobar" + MangoDatabase.create_index(unquote(db), ["foo"], ddoc: ddocid_invalid) + MangoDatabase.create_index(unquote(db), ["foo", "bar"], ddoc: ddocid_valid) + selector = %{"foo" => %{"$gt" => nil}} + + {:ok, resp_explain} = MangoDatabase.find(unquote(db), selector, sort: ["foo", "bar"], use_index: ddocid_invalid, explain: true) + assert resp_explain["index"]["ddoc"] == ddocid_valid + + {:ok, resp} = MangoDatabase.find(unquote(db), selector, sort: ["foo", "bar"], use_index: ddocid_invalid, return_raw: true) + result = + resp["warning"] + |> String.split("\n") + |> Enum.at(0) + |> String.downcase() + expected = "#{ddocid_invalid} was not used because it does not contain a valid index for this query." + assert result == expected + assert Enum.empty?(resp["docs"]) + end + + test "prefer use index over optimal index" do + # index on ["company"] even though index on ["company", "manager"] is better + ddocid_preferred = "_design/testsuboptimal" + MangoDatabase.create_index(unquote(db), ["baz"], ddoc: ddocid_preferred) + MangoDatabase.create_index(unquote(db), ["baz", "bar"]) + selector = %{"baz" => %{"$gt" => nil}, "bar" => %{"$gt" => nil}} + {:ok, resp} = MangoDatabase.find(unquote(db), selector, use_index: ddocid_preferred, return_raw: true) + assert not Map.has_key?(resp, "warning") + + {:ok, resp_explain} = MangoDatabase.find(unquote(db), selector, use_index: ddocid_preferred, explain: true) + assert resp_explain["index"]["ddoc"] == ddocid_preferred + end + + # This doc will not be saved given the new ddoc validation code + # in couch_mrview + test "manual bad view idx01" do + design_doc = %{ + "_id" => "_design/bad_view_index", + "language" => "query", + "views" => %{ + "queryidx1" => %{ + "map" => %{"fields" => %{"age" => "asc"}}, + "reduce" => "_count", + "options" => %{"def" => %{"fields" => [%{"age" => "asc"}]}, "w" => 2}, + } + }, + "views" => %{ + "views001" => %{ + "map" => "function(employee){if(employee.training)" + <> "{emit(employee.number, employee.training);}}" + } + }, + } + # fails if the result does not match the pattern {:error, _} + {:error, _} = MangoDatabase.save_docs(unquote(db), [design_doc]) + end + + test "explain sort reverse" do + selector = %{"manager" => %{"$gt" => nil}} + {:ok, resp_explain} = MangoDatabase.find(unquote(db), selector, fields: ["manager"], sort: [%{"manager" => "desc"}], explain: true) + assert resp_explain["index"]["type"] == "json" + end + + test "use index with invalid name" do + Enum.each(["foo/bar/baz", ["foo", "bar", "baz"]], fn index -> + {:error, resp} = MangoDatabase.find(unquote(db), %{"manager" => true}, use_index: index) + assert resp.status_code == 400 + end) + end + + test "use index without fallback succeeds for valid index" do + {:ok, docs} = MangoDatabase.find(unquote(db), %{"manager" => true}, use_index: "manager", allow_fallback: false) + assert not Enum.empty?(docs) + end + + test "use index without fallback fails for invalid index with fallback available" do + {:error, resp} = MangoDatabase.find(unquote(db), %{"manager" => true}, use_index: "invalid", allow_fallback: false) + assert resp.status_code == 400 + end + + test "use index without fallback succeeds for empty index" do + {:ok, docs} = MangoDatabase.find(unquote(db), %{"manager" => true}, use_index: [], allow_fallback: false) + assert not Enum.empty?(docs) + end + + test "use index without fallback fails for empty index" do + {:error, resp} = MangoDatabase.find(unquote(db), %{"company" => "foobar"}, use_index: [], allow_fallback: false) + assert resp.status_code == 400 + end + + test "use index without fallback fails for invalid index no fallback exists" do + {:error, resp} = MangoDatabase.find(unquote(db), %{"company" => "foobar"}, use_index: "invalid", allow_fallback: false) + assert resp.status_code == 400 + end + + test "index without fallback" do + {:ok, docs} = MangoDatabase.find(unquote(db), %{"manager" => true}, allow_fallback: false) + assert not Enum.empty?(docs) + end + + test "no index without fallback" do + {:error, resp} = MangoDatabase.find(unquote(db), %{"company" => "foobar"}, allow_fallback: false) + assert resp.status_code == 400 + end + end + end +end + +defmodule JSONIndexSelectionTest do + use CouchTestCase + require IndexSelectionTest + + @db_name "json-index-selection" + + setup do + UserDocs.setup(@db_name) + end + + IndexSelectionTest.describe(@db_name) + + test "uses all docs when fields do not match selector" do + # index exists on ["company", "manager"] but not ["company"] + # so we should fall back to all docs (so we include docs + # with no "manager" field) + selector = %{"company" => "Pharmex"} + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert length(docs) == 1 + assert Enum.at(docs, 0)["company"] == "Pharmex" + assert not Map.has_key?(Enum.at(docs, 0), "manager") + + {:ok, resp_explain} = MangoDatabase.find(@db_name, selector, explain: true) + assert resp_explain["index"]["type"] == "special" + end + + test "uses all docs when selector doesnt require fields to exist" do + # as in test above, use a selector that doesn't overlap with the index + # due to an explicit exists clause + selector = %{"company" => "Pharmex", "manager" => %{"$exists" => false}} + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert length(docs) == 1 + assert Enum.at(docs, 0)["company"] == "Pharmex" + assert not Map.has_key?(Enum.at(docs, 0), "manager") + + {:ok, resp_explain} = MangoDatabase.find(@db_name, selector, explain: true) + assert resp_explain["index"]["type"] == "special" + end +end + +defmodule TextIndexSelectionTest do + use CouchTestCase + + @db_name "text-index-selection" + + setup do + UserDocs.setup(@db_name, "text") + end + + test "with text" do + {:ok, resp} = MangoDatabase.find(@db_name, + %{ + "$text" => "Stephanie", + "name.first" => "Stephanie", + "name.last" => "This doesn't have to match anything.", + }, + explain: true + ) + assert resp["index"]["type"] == "text" + end + + test "no view index" do + {:ok, resp} = MangoDatabase.find(@db_name, %{"name.first" => "Ohai!"}, explain: true) + assert resp["index"]["type"] == "text" + end + + test "with or" do + {:ok, resp} = MangoDatabase.find(@db_name, + %{ + "$or" => [ + %{"name.first" => "Stephanie"}, + %{"name.last" => "This doesn't have to match anything."}, + ] + }, + explain: true + ) + assert resp["index"]["type"] == "text" + end + + test "manual bad text idx" do + design_doc = %{ + "_id" => "_design/bad_text_index", + "language" => "query", + "indexes" => %{ + "text_index" => %{ + "default_analyzer" => "keyword", + "default_field" => %{}, + "selector" => %{}, + "fields" => "all_fields", + "analyzer" => %{ + "name" => "perfield", + "default" => "keyword", + "fields" => %{"$default" => "standard"}, + }, + } + }, + "indexes" => %{ + "st_index" => %{ + "analyzer" => "standard", + "index" => "function(doc){\n index(\"st_index\", doc.geometry);\n}", + } + }, + } + MangoDatabase.save_docs(@db_name, [design_doc]) + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => 48}) + assert length(docs) == 1 + assert Enum.at(docs, 0)["name"]["first"] == "Stephanie" + assert Enum.at(docs, 0)["age"] == 48 + end +end + +defmodule MultiTextIndexSelectionTest do + use CouchTestCase + + @db_name "multi-text-index-selection" + + setup do + UserDocs.setup(@db_name) + MangoDatabase.create_text_index(@db_name, ddoc: "foo", analyzer: "keyword") + MangoDatabase.create_text_index(@db_name, ddoc: "bar", analyzer: "email") + :ok + end + + test "fallback to json with multi text" do + {:ok, resp} = MangoDatabase.find(@db_name, + %{"name.first" => "A first name", "name.last" => "A last name"}, + explain: true) + assert resp["index"]["type"] == "json" + end + + test "multi text index is error" do + {:error, resp} = MangoDatabase.find(@db_name, %{"$text" => "a query"}, explain: true) + assert resp.status_code == 400 + end + + test "use index works" do + {:ok, resp} = MangoDatabase.find(@db_name, %{"$text" => "a query"}, use_index: "foo", explain: true) + assert resp["index"]["ddoc"] == "_design/foo" + end +end + +defmodule RegexVsTextIndexTest do + use CouchTestCase + + @db_name "regex-text-index" + + setup do + MangoDatabase.recreate(@db_name) + :ok + end + + test "regex works with text index" do + doc = %{"currency" => "HUF", "location" => "EUROPE"} + saved_docs = MangoDatabase.save_docs(@db_name, [doc], w: 3) + + selector = %{"currency" => %{"$regex" => "HUF"}} + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert docs == saved_docs + + # Now that it is confirmed to be working, try again the + # previous query with a text index on `location`. This + # attempt should succeed as well. + MangoDatabase.create_text_index(@db_name, name: "TextIndexByLocation", fields: [%{"name" => "location", "type" => "string"}]) + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert docs == saved_docs + end +end diff --git a/test/elixir/test/mango/06-text-default-field-test.exs b/test/elixir/test/mango/06-text-default-field-test.exs new file mode 100644 index 00000000000..92c8a8b67f1 --- /dev/null +++ b/test/elixir/test/mango/06-text-default-field-test.exs @@ -0,0 +1,100 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule NoDefaultFieldTest do + use CouchTestCase + + @db_name "no-default-field" + + setup do + UserDocs.setup(@db_name, "text") + MangoDatabase.create_text_index( + @db_name, + default_field: false, + ddoc: "text" + ) + :ok + end + + test "basic" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Ramona"}, use_index: "text") + # Or should this throw an error? + assert Enum.empty?(docs) + end + + test "other fields exist" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => 22}, use_index: "text") + assert length(docs) == 1 + assert Enum.at(docs, 0)["user_id"] == 9 + end +end + +defmodule NoDefaultFieldWithAnalyzer do + use CouchTestCase + + @db_name "no-default-field-with-analyzer" + + setup do + UserDocs.setup(@db_name, "text") + MangoDatabase.create_text_index( + @db_name, + default_field: %{"enabled" => false, "analyzer" => "keyword"}, + ddoc: "text" + ) + :ok + end + + test "basic" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Ramona"}, use_index: "text") + assert Enum.empty?(docs) + end + + test "other fields exist" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => 22}, use_index: "text") + assert length(docs) == 1 + assert Enum.at(docs, 0)["user_id"] == 9 + end +end + +defmodule DefaultFieldWithCustomAnalyzer do + use CouchTestCase + + @db_name "default-field-with-custom-analyser" + + setup do + UserDocs.setup(@db_name, "text") + MangoDatabase.create_text_index( + @db_name, + default_field: %{"enabled" => true, "analyzer" => "keyword"}, + ddoc: "text" + ) + :ok + end + + test "basic" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Ramona"}, use_index: "text") + assert length(docs) == 1 + assert Enum.at(docs, 0)["user_id"] == 9 + end + + test "not analyzed" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Lott Place"}, use_index: "text") + assert length(docs) == 1 + assert Enum.at(docs, 0)["user_id"] == 9 + + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Lott"}, use_index: "text") + assert Enum.empty?(docs) + + {:ok, docs} = MangoDatabase.find(@db_name, %{"$text" => "Place"}, use_index: "text") + assert Enum.empty?(docs) + end +end diff --git a/test/elixir/test/mango/06_basic_text_test.exs b/test/elixir/test/mango/06_basic_text_test.exs new file mode 100644 index 00000000000..08c39c6313e --- /dev/null +++ b/test/elixir/test/mango/06_basic_text_test.exs @@ -0,0 +1,91 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule BasicTextTest do + use CouchTestCase + + @db_name "basic-text" + + setup do + UserDocs.setup(@db_name, "text") + end + + test "explain options" do + {:ok, explain} = MangoDatabase.find( + @db_name, + %{"age" => %{"$gt" => 0}}, + fields: ["manager"], + allow_fallback: false, + explain: true + ) + opts = explain["opts"] + assert opts["r"] == 1 + assert opts["limit"] == 25 + assert opts["skip"] == 0 + assert opts["fields"] == ["manager"] + assert opts["sort"] == %{} + assert opts["bookmark"] == "nil" + assert opts["conflicts"] == false + assert opts["execution_stats"] == false + assert opts["partition"] == "" + assert opts["stable"] == false + assert opts["stale"] == false + assert opts["update"] == true + assert opts["use_index"] == [] + assert opts["allow_fallback"] == false + end + + test "explain with bookmarks" do + query = %{"age" => %{"$gt" => 42}} + + {:ok, resp} = MangoDatabase.find( + @db_name, + query, + limit: 1, + allow_fallback: false, + return_raw: true + ) + assert length(resp["docs"]) == 1 + assert resp["bookmark"] != "nil" + {:ok, explain} = MangoDatabase.find( + @db_name, + query, + bookmark: resp["bookmark"], + allow_fallback: false, + explain: true + ) + assert is_binary(explain["opts"]["bookmark"]) + assert resp["bookmark"] == explain["opts"]["bookmark"] + end +end + +defmodule ElemMatchTests do + use CouchTestCase + + @db_name "basic-text-elem-match" + + setup do + FriendDocs.setup(@db_name, "text") + end + + test "elem match non object" do + q = %{"bestfriends" => %{"$elemMatch" => %{"$eq" => "Wolverine", "$eq" => "Cyclops"}}} + {:ok, docs} = MangoDatabase.find(@db_name, q) + assert length(docs) == 1 + assert Enum.at(docs, 0)["bestfriends"] == ["Wolverine", "Cyclops"] + + q = %{"results" => %{"$elemMatch" => %{"$gte" => 80, "$lt" => 85}}} + {:ok, docs} = MangoDatabase.find(@db_name, q) + assert length(docs) == 1 + assert Enum.at(docs, 0)["results"] == [82, 85, 88] + end +end diff --git a/src/couch/test/eunit/fixtures/os_daemon_die_on_boot.sh b/test/elixir/test/mango/08_text_limit_test.exs old mode 100755 new mode 100644 similarity index 57% rename from src/couch/test/eunit/fixtures/os_daemon_die_on_boot.sh rename to test/elixir/test/mango/08_text_limit_test.exs index 256ee793596..07c6b01f98d --- a/src/couch/test/eunit/fixtures/os_daemon_die_on_boot.sh +++ b/test/elixir/test/mango/08_text_limit_test.exs @@ -1,15 +1,29 @@ -#!/bin/sh -e -# # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of # the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -exit 1 +defmodule LimitTests do + use CouchTestCase + + @db_name "limit-docs" + + setup do + LimitDocs.setup(@db_name, "text") + end + + test "limit field" do + q = %{"$or" => [%{"user_id" => %{"$lt" => 10}}, %{"filtered_array.[]" => 1}]} + {:ok, docs} = MangoDatabase.find(@db_name, q, limit: 10) + + assert length(docs) == 8 + Enum.each(docs, fn d -> assert d["user_id"] < 10 end) + end +end diff --git a/test/elixir/test/mango/11_ignore_design_docs_test.exs b/test/elixir/test/mango/11_ignore_design_docs_test.exs new file mode 100644 index 00000000000..b6d90edcbde --- /dev/null +++ b/test/elixir/test/mango/11_ignore_design_docs_test.exs @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule IgnoreDesignDocsForAllDocsIndexTests do + use CouchTestCase + + @db_name "ignore-design-docs" + + setup do + MangoDatabase.recreate(@db_name) + docs = [ + %{"_id" => "_design/my-design-doc"}, + %{"_id" => "54af50626de419f5109c962f", "user_id" => 0, "age" => 10, "name" => "Jimi"}, + %{"_id" => "54af50622071121b25402dc3", "user_id" => 1, "age" => 11, "name" => "Eddie"} + ] + MangoDatabase.save_docs(@db_name, docs) + :ok + end + + test "should not return design docs" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"_id" => %{"$gte" => nil}}) + assert length(docs) == 2 + end +end diff --git a/test/elixir/test/mango/12_use_correct_index_test.exs b/test/elixir/test/mango/12_use_correct_index_test.exs new file mode 100644 index 00000000000..627cc86cb7d --- /dev/null +++ b/test/elixir/test/mango/12_use_correct_index_test.exs @@ -0,0 +1,139 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule ChooseCorrectIndexForDocs do + use CouchTestCase + + @db_name "choose-correct-index" + @docs [ + %{"_id" => "_design/my-design-doc"}, + %{ + "_id" => "54af50626de419f5109c962f", + "user_id" => 0, + "age" => 10, + "name" => "Jimi", + "location" => "UK", + "number" => 4, + }, + %{ + "_id" => "54af50622071121b25402dc3", + "user_id" => 1, + "age" => 12, + "name" => "Eddie", + "location" => "ZAR", + "number" => 2, + }, + %{ + "_id" => "54af50622071121b25402dc6", + "user_id" => 1, + "age" => 6, + "name" => "Harry", + "location" => "US", + "number" => 8, + }, + %{ + "_id" => "54af50622071121b25402dc9", + "name" => "Eddie", + "occupation" => "engineer", + "number" => 7, + }, + ] + @docs2 [%{"a" => 1, "b" => 1, "c" => 1}, %{"a" => 1000, "d" => 1000, "e" => 1000}] + + setup do + MangoDatabase.recreate(@db_name) + MangoDatabase.save_docs(@db_name, @docs) + :ok + end + + test "choose index with one field in index" do + MangoDatabase.create_index(@db_name, ["name", "age", "user_id"], ddoc: "aaa") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "zzz") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie"}, explain: true) + assert explain["index"]["ddoc"] == "_design/zzz" + end + + test "choose index with two" do + MangoDatabase.create_index(@db_name, ["name", "age", "user_id"], ddoc: "aaa") + MangoDatabase.create_index(@db_name, ["name", "age"], ddoc: "bbb") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "zzz") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie", "age" => %{"$gte" => 12}}, explain: true) + assert explain["index"]["ddoc"] == "_design/bbb" + end + + test "choose index alphabetically" do + MangoDatabase.create_index(@db_name, ["name"], ddoc: "aaa") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "bbb") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "zzz") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie", "age" => %{"$gte" => 12}}, explain: true) + assert explain["index"]["ddoc"] == "_design/aaa" + end + + test "choose index most accurate" do + MangoDatabase.create_index(@db_name, ["name", "age", "user_id"], ddoc: "aaa") + MangoDatabase.create_index(@db_name, ["name", "age"], ddoc: "bbb") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "zzz") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie", "age" => %{"$gte" => 12}}, explain: true) + assert explain["index"]["ddoc"] == "_design/bbb" + end + + test "choose index most accurate in memory selector" do + MangoDatabase.create_index(@db_name, ["name", "location", "user_id"], ddoc: "aaa") + MangoDatabase.create_index(@db_name, ["name", "age", "user_id"], ddoc: "bbb") + MangoDatabase.create_index(@db_name, ["name"], ddoc: "zzz") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie", "number" => %{"$lte" => 12}}, explain: true) + assert explain["index"]["ddoc"] == "_design/zzz" + end + + test "warn on full db scan" do + selector = %{"not_indexed" => "foo"} + {:ok, explain} = MangoDatabase.find(@db_name, selector, explain: true, return_raw: true) + assert explain["index"]["type"] == "special" + {:ok, resp} = MangoDatabase.find(@db_name, selector, return_raw: true) + assert resp["warning"] == "No matching index found, create an index to optimize query time." + end + + test "chooses idxA" do + MangoDatabase.save_docs(@db_name, @docs2) + MangoDatabase.create_index(@db_name, ["a", "b", "c"]) + MangoDatabase.create_index(@db_name, ["a", "d", "e"]) + {:ok, explain} = MangoDatabase.find(@db_name, + %{"a" => %{"$gt" => 0}, "b" => %{"$gt" => 0}, "c" => %{"$gt" => 0}}, explain: true + ) + assert explain["index"]["def"]["fields"] == [%{"a" => "asc"}, %{"b" => "asc"}, %{"c" => "asc"}] + end + + test "can query with range on secondary column" do + MangoDatabase.create_index(@db_name, ["age", "name"], ddoc: "bbb") + selector = %{"age" => 10, "name" => %{"$gte" => 0}} + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert length(docs) == 1 + {:ok, explain} = MangoDatabase.find(@db_name, selector, explain: true) + assert explain["index"]["ddoc"] == "_design/bbb" + assert explain["mrargs"]["end_key"] == [10, ""] + end + + # all documents contain an _id and _rev field they + # should not be used to restrict indexes based on the + # fields required by the selector + test "choose index with id" do + MangoDatabase.create_index(@db_name, ["name", "_id"], ddoc: "aaa") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie"}, explain: true) + assert explain["index"]["ddoc"] == "_design/aaa" + end + + test "choose index with rev" do + MangoDatabase.create_index(@db_name, ["name", "_rev"], ddoc: "aaa") + {:ok, explain} = MangoDatabase.find(@db_name, %{"name" => "Eddie"}, explain: true) + assert explain["index"]["ddoc"] == "_design/aaa" + end +end diff --git a/test/elixir/test/mango/14_json_pagination_test.exs b/test/elixir/test/mango/14_json_pagination_test.exs new file mode 100644 index 00000000000..6a8dc0883f8 --- /dev/null +++ b/test/elixir/test/mango/14_json_pagination_test.exs @@ -0,0 +1,271 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule PaginateJsonDocs do + use CouchTestCase + + @db_name "paginate-json-docs" + @docs [ + %{"_id" => "100", "name" => "Jimi", "location" => "AUS", "user_id" => 1, "same" => "value"}, + %{"_id" => "200", "name" => "Eddie", "location" => "BRA", "user_id" => 2, "same" => "value"}, + %{"_id" => "300", "name" => "Harry", "location" => "CAN", "user_id" => 3, "same" => "value"}, + %{"_id" => "400", "name" => "Eddie", "location" => "DEN", "user_id" => 4, "same" => "value"}, + %{"_id" => "500", "name" => "Jones", "location" => "ETH", "user_id" => 5, "same" => "value"}, + %{ + "_id" => "600", + "name" => "Winnifried", + "location" => "FRA", + "user_id" => 6, + "same" => "value", + }, + %{"_id" => "700", "name" => "Marilyn", "location" => "GHA", "user_id" => 7, "same" => "value"}, + %{"_id" => "800", "name" => "Sandra", "location" => "ZAR", "user_id" => 8, "same" => "value"}, + ] + + setup do + MangoDatabase.recreate(@db_name) + MangoDatabase.save_docs(@db_name, @docs) + :ok + end + + test "all docs paginate to end" do + selector = %{"_id" => %{"$gt" => 0}} + # Page 1 + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], limit: 5, return_raw: true) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.at(docs, 0)["_id"] == "100" + assert length(docs) == 5 + + # Page 2 + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], bookmark: bookmark, limit: 5, return_raw: true + ) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.at(docs, 0)["_id"] == "600" + assert length(docs) == 3 + + # Page 3 + {:ok, resp} = MangoDatabase.find(@db_name, selector, bookmark: bookmark, limit: 5, return_raw: true) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "return previous bookmark for empty" do + selector = %{"_id" => %{"$gt" => 0}} + # Page 1 + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], return_raw: true) + bookmark1 = resp["bookmark"] + docs = resp["docs"] + assert length(docs) == 8 + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], return_raw: true, bookmark: bookmark1 + ) + bookmark2 = resp["bookmark"] + docs = resp["docs"] + assert Enum.empty?(docs) + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], return_raw: true, bookmark: bookmark2 + ) + bookmark3 = resp["bookmark"] + docs = resp["docs"] + assert bookmark3 == bookmark2 + assert Enum.empty?(docs) + end + + test "all docs with skip" do + selector = %{"_id" => %{"$gt" => 0}} + # Page 1 + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], skip: 2, limit: 5, return_raw: true) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.at(docs, 0)["_id"] == "300" + assert length(docs) == 5 + + # Page 2 + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], bookmark: bookmark, limit: 5, return_raw: true + ) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.at(docs, 0)["_id"] == "800" + assert length(docs) == 1 + {:ok, resp} = MangoDatabase.find(@db_name, selector, bookmark: bookmark, limit: 5, return_raw: true) + bookmark = resp["bookmark"] + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "all docs reverse" do + selector = %{"_id" => %{"$gt" => 0}} + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], sort: [%{"_id" => "desc"}], limit: 5, return_raw: true + ) + docs = resp["docs"] + bookmark1 = resp["bookmark"] + assert length(docs) == 5 + assert Enum.at(docs, 0)["_id"] == "800" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, + fields: ["_id"], + sort: [%{"_id" => "desc"}], + limit: 5, + return_raw: true, + bookmark: bookmark1 + ) + docs = resp["docs"] + bookmark2 = resp["bookmark"] + assert length(docs) == 3 + assert Enum.at(docs, 0)["_id"] == "300" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, + fields: ["_id"], + sort: [%{"_id" => "desc"}], + limit: 5, + return_raw: true, + bookmark: bookmark2 + ) + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "bad bookmark" do + {:error, response} = MangoDatabase.find(@db_name, %{"_id" => %{"$gt" => 0}}, bookmark: "bad-bookmark") + assert response.body["error"] == "invalid_bookmark" + assert response.body["reason"] == "Invalid bookmark value: \"bad-bookmark\"" + assert response.status_code == 400 + end + + test "throws error on text bookmark" do + bookmark = ( + "g2wAAAABaANkABFub2RlMUBjb3VjaGRiLm5ldGwAAAACYQBiP____2poAkY_8AAAAAAAAGEHag" + ) + {:error, response} = MangoDatabase.find(@db_name, %{"_id" => %{"$gt" => 0}}, bookmark: bookmark) + assert response.body["error"] == "invalid_bookmark" + assert response.status_code == 400 + end + + test "index pagination" do + MangoDatabase.create_index(@db_name, ["location"]) + selector = %{"location" => %{"$gt" => "A"}} + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], limit: 5, return_raw: true) + docs = resp["docs"] + bookmark1 = resp["bookmark"] + assert length(docs) == 5 + assert Enum.at(docs, 0)["_id"] == "100" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark1 + ) + docs = resp["docs"] + bookmark2 = resp["bookmark"] + assert length(docs) == 3 + assert Enum.at(docs, 0)["_id"] == "600" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark2 + ) + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "index pagination two keys" do + MangoDatabase.create_index(@db_name, ["location", "user_id"]) + selector = %{"location" => %{"$gt" => "A"}, "user_id" => %{"$gte" => 1}} + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], limit: 5, return_raw: true) + docs = resp["docs"] + bookmark1 = resp["bookmark"] + assert length(docs) == 5 + assert Enum.at(docs, 0)["_id"] == "100" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark1 + ) + docs = resp["docs"] + bookmark2 = resp["bookmark"] + assert length(docs) == 3 + assert Enum.at(docs, 0)["_id"] == "600" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark2 + ) + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "index pagination reverse" do + MangoDatabase.create_index(@db_name, ["location", "user_id"]) + selector = %{"location" => %{"$gt" => "A"}, "user_id" => %{"$gte" => 1}} + sort = [%{"location" => "desc"}, %{"user_id" => "desc"}] + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], sort: sort, limit: 5, return_raw: true + ) + docs = resp["docs"] + bookmark1 = resp["bookmark"] + assert length(docs) == 5 + assert Enum.at(docs, 0)["_id"] == "800" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, + fields: ["_id"], + limit: 5, + sort: sort, + return_raw: true, + bookmark: bookmark1 + ) + docs = resp["docs"] + bookmark2 = resp["bookmark"] + assert length(docs) == 3 + assert Enum.at(docs, 0)["_id"] == "300" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, + fields: ["_id"], + limit: 5, + sort: sort, + return_raw: true, + bookmark: bookmark2 + ) + docs = resp["docs"] + assert Enum.empty?(docs) + end + + test "index pagination same emitted key" do + MangoDatabase.create_index(@db_name, ["same"]) + selector = %{"same" => %{"$gt" => ""}} + {:ok, resp} = MangoDatabase.find(@db_name, selector, fields: ["_id"], limit: 5, return_raw: true) + docs = resp["docs"] + bookmark1 = resp["bookmark"] + assert length(docs) == 5 + assert Enum.at(docs, 0)["_id"] == "100" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark1 + ) + docs = resp["docs"] + bookmark2 = resp["bookmark"] + assert length(docs) == 3 + assert Enum.at(docs, 0)["_id"] == "600" + + {:ok, resp} = MangoDatabase.find(@db_name, + selector, fields: ["_id"], limit: 5, return_raw: true, bookmark: bookmark2 + ) + docs = resp["docs"] + assert Enum.empty?(docs) + end +end diff --git a/test/elixir/test/mango/22-covering-index-test.exs b/test/elixir/test/mango/22-covering-index-test.exs new file mode 100644 index 00000000000..eda0ded4abb --- /dev/null +++ b/test/elixir/test/mango/22-covering-index-test.exs @@ -0,0 +1,230 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule CoveringIndexTest do + use CouchTestCase + defmacro describe(db) do + quote do + test "index covers query 1 field index id" do + covered?(unquote(db), %{"age" => %{"$gte" => 32}}, ["_id"], "age") + end + + test "index covers query 2 field index id" do + covered?( + unquote(db), %{"company" => "Lyria", "manager" => true}, ["_id"], "company_and_manager" + ) + end + + test "index covers query 2 field index extract field" do + covered?( + unquote(db), + %{"company" => %{"$exists" => true}, "manager" => true}, + ["company"], + "company_and_manager" + ) + end + + test "index covers query 2 field index extract field force index" do + covered?( + unquote(db), + %{"company" => %{"$exists" => true}, "manager" => true}, + ["company"], + "company_and_manager", + use_index: "company_and_manager" + ) + end + + test "index covers query elemMatch" do + covered?( + unquote(db), %{"favorites" => %{"$elemMatch" => %{"$eq" => "Erlang"}}}, ["favorites"], "favorites" + ) + end + + test "index covers query composite field_id" do + covered?( + unquote(db), %{"name" => %{"first" => "Stephanie", "last" => "Kirkland"}}, ["_id"], "name" + ) + end + + test "index does not cover query empty selector" do + not_covered?(unquote(db), %{}, ["_id"]) + end + + test "index does not cover query field not in index" do + not_covered?(unquote(db), %{"age" => %{"$gte" => 32}}, ["name"]) + end + + test "index does not cover query all fields" do + not_covered?(unquote(db), %{"age" => %{"$gte" => 32}}, []) + end + + test "index does not cover query partial selector id" do + not_covered?(unquote(db), %{"location.state" => "Nevada"}, ["_id"]) + end + + test "index does not cover query partial selector" do + not_covered?(unquote(db), %{"name.last" => "Hernandez"}, ["name.first"]) + end + + test "index does not cover selector with more fields" do + not_covered?( + unquote(db), + %{ + "$and" => [ + %{"age" => %{"$ne" => 23}}, + %{"twitter" => %{"$not" => %{"$regex" => "^@.*[0-9]+$"}}}, + %{"location.address.number" => %{"$gt" => 4288}}, + %{"location.city" => %{"$ne" => "Pico Rivera"}}, + ] + }, + ["twitter"], + use_index: "twitter" + ) + end + end + end +end + +defmodule RegularCoveringIndexTest do + use CouchTestCase + require CoveringIndexTest + + @db_name "regular-covering-index" + + setup do + UserDocs.setup(@db_name) + end + + def covered?(db, selector, fields, index, opts \\ []) do + use_index = Keyword.get(opts, :use_index, nil) + {:ok, resp} = MangoDatabase.find(db, selector, fields: fields, use_index: use_index, explain: true) + + assert resp["index"]["type"] == "json" + assert resp["index"]["name"] == index + assert resp["mrargs"]["include_docs"] == false + assert resp["covering"] == true + end + + def not_covered?(db, selector, fields, opts \\ []) do + use_index = Keyword.get(opts, :use_index, nil) + {:ok, resp} = MangoDatabase.find(db, selector, fields: fields, use_index: use_index, explain: true) + + assert resp["mrargs"]["include_docs"] == true + assert resp["covering"] == false + end + + CoveringIndexTest.describe(@db_name) + + test "covering index provides correct answer 2 field index" do + {:ok, docs} = MangoDatabase.find( + @db_name, + %{"company" => %{"$exists" => true}, "manager" => true}, + sort: [%{"company" => "asc"}], + fields: ["company"], + use_index: "company_and_manager" + ) + expected = [ + %{"company" => "Affluex"}, + %{"company" => "Globoil"}, + %{"company" => "Lyria"}, + %{"company" => "Manglo"}, + %{"company" => "Myopium"}, + %{"company" => "Niquent"}, + %{"company" => "Oulu"}, + %{"company" => "Prosely"}, + %{"company" => "Tasmania"}, + %{"company" => "Zialactic"}, + ] + assert docs == expected + end + + test "covering index provides correct answer id" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gte" => 32}}, fields: ["_id"]) + expected = [ + %{"_id" => "659d0430-b1f4-413a-a6b7-9ea1ef071325"}, + %{"_id" => "48ca0455-8bd0-473f-9ae2-459e42e3edd1"}, + %{"_id" => "e900001d-bc48-48a6-9b1a-ac9a1f5d1a03"}, + %{"_id" => "b31dad3f-ae8b-4f86-8327-dfe8770beb27"}, + %{"_id" => "71562648-6acb-42bc-a182-df6b1f005b09"}, + %{"_id" => "c78c529f-0b07-4947-90a6-d6b7ca81da62"}, + %{"_id" => "8e1c90c0-ac18-4832-8081-40d14325bde0"}, + %{"_id" => "6c0afcf1-e57e-421d-a03d-0c0717ebf843"}, + %{"_id" => "5b61abc1-a3d3-4092-b9d7-ced90e675536"}, + %{"_id" => "a33d5457-741a-4dce-a217-3eab28b24e3e"}, + %{"_id" => "b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4"}, + %{"_id" => "b1e70402-8add-4068-af8f-b4f3d0feb049"}, + %{"_id" => "0461444c-e60a-457d-a4bb-b8d811853f21"}, + ] + assert docs == expected + end +end + +defmodule PartitionedCoveringIndexTest do + use CouchTestCase + require CoveringIndexTest + + @db_name "partitioned-covering-index" + + setup do + UserDocs.setup(@db_name, "view", true) + end + + def covered?(db, selector, fields, index, opts \\ []) do + use_index = Keyword.get(opts, :use_index, nil) + {:ok, resp} = MangoDatabase.find(db, selector, fields: fields, use_index: use_index, explain: true, partition: "0") + + assert resp["index"]["type"] == "json" + assert resp["index"]["name"] == index + assert resp["mrargs"]["include_docs"] == false + assert resp["covering"] == true + end + + def not_covered?(db, selector, fields, opts \\ []) do + use_index = Keyword.get(opts, :use_index, nil) + {:ok, resp} = MangoDatabase.find(db, selector, fields: fields, use_index: use_index, explain: true, partition: "0") + + assert resp["mrargs"]["include_docs"] == true + assert resp["covering"] == false + end + + CoveringIndexTest.describe(@db_name) + + test "covering index provides correct answer 2 field index" do + {:ok, docs} = MangoDatabase.find( + @db_name, + %{"company" => %{"$exists" => true}, "manager" => true}, + sort: [%{"company" => "asc"}], + fields: ["company"], + use_index: "company_and_manager", + partition: "0" + ) + expected = [ + %{"company" => "Manglo"}, + %{"company" => "Oulu"}, + %{"company" => "Prosely"}, + %{"company" => "Tasmania"}, + ] + assert docs == expected + end + + test "covering index provides correct answer id" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$gte" => 32}}, fields: ["_id"], partition: "0") + + expected = [ + %{"_id" => "0:0461444c-e60a-457d-a4bb-b8d811853f21"}, + %{"_id" => "0:5b61abc1-a3d3-4092-b9d7-ced90e675536"}, + %{"_id" => "0:71562648-6acb-42bc-a182-df6b1f005b09"}, + %{"_id" => "0:b31dad3f-ae8b-4f86-8327-dfe8770beb27"}, + ] + assert Enum.sort_by(docs, fn x -> x["_id"] end) == expected + end +end diff --git a/test/elixir/test/mango/25_beginswith_test.exs b/test/elixir/test/mango/25_beginswith_test.exs new file mode 100644 index 00000000000..e14c01093bd --- /dev/null +++ b/test/elixir/test/mango/25_beginswith_test.exs @@ -0,0 +1,143 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule BeginsWithOperator do + use CouchTestCase + + @db_name "begins-with-operator" + + @docs [ + %{"_id" => "aaa", "name" => "Jimi", "location" => "AUS", "age" => 27}, + %{"_id" => "abc", "name" => "Eddie", "location" => "AND", "age" => 65}, + %{"_id" => "bbb", "name" => "Harry", "location" => "CAN", "age" => 21}, + %{"_id" => "ccc", "name" => "Eddie", "location" => "DEN", "age" => 37}, + %{"_id" => "ddd", "name" => "Jones", "location" => "ETH", "age" => 49}, +] + + setup do + MangoDatabase.recreate(@db_name) + MangoDatabase.save_docs(@db_name, @docs) + MangoDatabase.create_index(@db_name, ["location"]) + MangoDatabase.create_index(@db_name, ["name", "location"]) + :ok + end + + # split into Unicode graphemes + defp split_unicode(binary) do + binary + |> String.graphemes() + end + + defp get_mrargs(selector, opts \\ []) do + sort = Keyword.get(opts, :sort, []) + {:ok, explain} = MangoDatabase.find(@db_name, selector, sort: sort, explain: true) + explain["mrargs"] + end + + defp assert_doc_ids(user_ids, docs) do + user_ids_returned = + docs + |> Enum.map(fn d -> d["_id"] end) + |> Enum.sort() + assert Enum.sort(user_ids) == user_ids_returned + end + + test "basic" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"location" => %{"$beginsWith" => "A"}}) + assert length(docs) == 2 + assert_doc_ids(["aaa", "abc"], docs) + end + + test "json range" do + mrargs = get_mrargs(%{"location" => %{"$beginsWith" => "A"}}) + assert mrargs["start_key"] == ["A"] + assert mrargs["end_key"] == [<<"A\xef\xbf\xbf">>, <<"">>] + end + + test "compound key" do + selector = %{"name" => "Eddie", "location" => %{"$beginsWith" => "A"}} + mrargs = get_mrargs(selector) + assert mrargs["start_key"] == ["Eddie", "A"] + assert mrargs["end_key"] == [<<"Eddie">>, <<"A\xef\xbf\xbf">>, <<"">>] + + {:ok, docs} = MangoDatabase.find(@db_name, selector) + assert length(docs) == 1 + assert_doc_ids(["abc"], docs) + end + + test "sort" do + selector = %{"location" => %{"$beginsWith" => "A"}} + cases = [ + %{ + "sort" => ["location"], + "start_key" => [<<"A">>], + "end_key" => [<<"A\xef\xbf\xbf">>, <<"">>], + "direction" => "fwd", + }, + %{ + "sort" => [%{"location" => "desc"}], + "start_key" => [<<"A\xef\xbf\xbf">>, <<"">>], + "end_key" => [<<"A">>], + "direction" => "rev", + }, + ] + for case <- cases do + mrargs = get_mrargs(selector, sort: case["sort"]) + + assert mrargs["start_key"] == case["start_key"] + assert mrargs["end_key"] == case["end_key"] + assert mrargs["direction"] == case["direction"] + end + end + + test "all docs range" do + mrargs = get_mrargs(%{"_id" => %{"$beginsWith" => "a"}}) + + assert mrargs["start_key"] == "a" + end_key_bytes = split_unicode(mrargs["end_key"]) + assert end_key_bytes == [<<"a">>, <<"\xef\xbf\xbf">>] + end + + test "no index" do + selector = %{"foo" => %{"$beginsWith" => "a"}} + {:ok, resp_explain} = MangoDatabase.find(@db_name, selector, explain: true) + mrargs = resp_explain["mrargs"] + + assert resp_explain["index"]["type"] == "special" + assert mrargs["start_key"] == nil + assert mrargs["end_key"] == "" + end + + test "invalid operand" do + {:error, resp} = MangoDatabase.find(@db_name, %{"_id" => %{"$beginsWith" => true}}) + assert resp.status_code == 400 + assert resp.body["error"] == "invalid_operator" + end + + test "does not match non string value" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"age" => %{"$beginsWith" => "a"}}) + assert Enum.empty?(docs) + end + + test "no matches" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"name" => %{"$beginsWith" => "Z"}}) + assert Enum.empty?(docs) + end + + test "case sensitivity" do + {:ok, docs} = MangoDatabase.find(@db_name, %{"name" => %{"$beginsWith" => "j"}}) + assert Enum.empty?(docs) + + {:ok, docs} = MangoDatabase.find(@db_name, %{"name" => %{"$beginsWith" => "J"}}) + assert length(docs) == 2 + end +end diff --git a/test/elixir/test/nouveau_test.exs b/test/elixir/test/nouveau_test.exs index 18732766db8..888513fabdc 100644 --- a/test/elixir/test/nouveau_test.exs +++ b/test/elixir/test/nouveau_test.exs @@ -140,7 +140,7 @@ defmodule NouveauTest do def assert_status_code(resp, code) do assert resp.status_code == code, - "status code: #{resp.status_code}, resp body: #{:jiffy.encode(resp.body)}" + "status code: #{resp.status_code}, resp body: #{:jiffy.encode(resp.body, [:use_nil])}" end test "user-agent header is forbidden", _context do diff --git a/test/elixir/test/partition_crud_test.exs b/test/elixir/test/partition_crud_test.exs index 7e32abbdc10..dba575da3e0 100644 --- a/test/elixir/test/partition_crud_test.exs +++ b/test/elixir/test/partition_crud_test.exs @@ -154,7 +154,7 @@ defmodule PartitionCrudTest do "error" => "illegal_docid", "id" => "my-partition-post", "reason" => "Doc id must be of form partition:id", - "rev" => :null + "rev" => nil } } ], diff --git a/test/elixir/test/partition_mango_test.exs b/test/elixir/test/partition_mango_test.exs index 9e4f1e7838a..c62ef3b0084 100644 --- a/test/elixir/test/partition_mango_test.exs +++ b/test/elixir/test/partition_mango_test.exs @@ -478,7 +478,7 @@ defmodule PartitionMangoTest do %{:body => body} = resp assert body["index"]["name"] == "_all_docs" - assert body["mrargs"]["partition"] == :null + assert body["mrargs"]["partition"] == nil resp = Couch.post( @@ -493,7 +493,7 @@ defmodule PartitionMangoTest do %{:body => body} = resp assert body["index"]["def"] == %{"fields" => [%{"some" => "asc"}]} - assert body["mrargs"]["partition"] == :null + assert body["mrargs"]["partition"] == nil end @tag :with_partitioned_db @@ -733,4 +733,118 @@ defmodule PartitionMangoTest do %{:body => %{"reason" => reason}} = resp assert Regex.match?(~r/No global index exists for this sort/, reason) end + + @tag :with_partitioned_db + test "explain options", context do + db_name = context[:db_name] + UserDocs.setup(db_name, "view", true) + + {:ok, explain} = MangoDatabase.find( + db_name, %{"age" => %{"$gt" => 0}}, + fields: ["manager"], partition: "0", explain: true + ) + opts = explain["opts"] + assert opts["r"] == 1 + assert opts["limit"] == 25 + assert opts["skip"] == 0 + assert opts["fields"] == ["manager"] + assert opts["sort"] == %{} + assert opts["bookmark"] == "nil" + assert opts["conflicts"] == false + assert opts["execution_stats"] == false + assert opts["partition"] == "0" + assert opts["stable"] == false + assert opts["stale"] == false + assert opts["update"] == true + assert opts["use_index"] == [] + assert opts["allow_fallback"] == true + end + + @tag :with_partitioned_db + test "explain options (text)", context do + db_name = context[:db_name] + UserDocs.setup(db_name, "text", true) + + {:ok, explain} = MangoDatabase.find( + db_name, + %{"age" => %{"$gt" => 0}}, + fields: ["manager"], + partition: "0", + allow_fallback: false, + explain: true + ) + opts = explain["opts"] + assert opts["r"] == 1 + assert opts["limit"] == 25 + assert opts["skip"] == 0 + assert opts["fields"] == ["manager"] + assert opts["sort"] == %{} + assert opts["bookmark"] == "nil" + assert opts["conflicts"] == false + assert opts["execution_stats"] == false + assert opts["partition"] == "0" + assert opts["stable"] == false + assert opts["stale"] == false + assert opts["update"] == true + assert opts["use_index"] == [] + assert opts["allow_fallback"] == false + end + + @tag :with_partitioned_db + test "explain works with bookmarks", context do + db_name = context[:db_name] + UserDocs.setup(db_name, "view", true) + + query = %{"age" => %{"$gt" => 42}} + partition = "0" + + {:ok, resp} = MangoDatabase.find( + db_name, + query, + partition: partition, + limit: 1, + return_raw: true + ) + assert length(resp["docs"]) == 1 + assert resp["bookmark"] != "nil" + {:ok, explain} = MangoDatabase.find( + db_name, + query, + partition: partition, + bookmark: resp["bookmark"], + explain: true + ) + assert is_binary(explain["opts"]["bookmark"]) + assert resp["bookmark"] == explain["opts"]["bookmark"] + end + + @tag :with_partitioned_db + test "explain works with bookmarks (text)", context do + db_name = context[:db_name] + UserDocs.setup(db_name, "text", true) + + query = %{"age" => %{"$gt" => 42}} + partition = "0" + + {:ok, resp} = MangoDatabase.find( + db_name, + query, + partition: partition, + limit: 1, + allow_fallback: false, + return_raw: true + ) + assert length(resp["docs"]) == 1 + assert resp["bookmark"] != "nil" + {:ok, explain} = MangoDatabase.find( + db_name, + query, + partition: partition, + bookmark: resp["bookmark"], + allow_fallback: false, + explain: true + ) + assert is_binary(explain["opts"]["bookmark"]) + assert resp["bookmark"] == explain["opts"]["bookmark"] + end end diff --git a/test/elixir/test/replication_test.exs b/test/elixir/test/replication_test.exs index 3151363243c..75e69ed2db5 100644 --- a/test/elixir/test/replication_test.exs +++ b/test/elixir/test/replication_test.exs @@ -1627,7 +1627,7 @@ defmodule ReplicationTest do end def set_security(db_name, sec_props) do - resp = Couch.put("/#{db_name}/_security", body: :jiffy.encode(sec_props)) + resp = Couch.put("/#{db_name}/_security", body: :jiffy.encode(sec_props, [:use_nil])) assert HTTPotion.Response.success?(resp) assert resp.body["ok"] end diff --git a/test/elixir/test/reshard_basic_test.exs b/test/elixir/test/reshard_basic_test.exs index dcb198c4689..5fe5f764383 100644 --- a/test/elixir/test/reshard_basic_test.exs +++ b/test/elixir/test/reshard_basic_test.exs @@ -24,12 +24,12 @@ defmodule ReshardBasicTest do test "basic api querying, no jobs present" do summary = get_summary() assert summary["state"] == "running" - assert summary["state_reason"] == :null + assert summary["state_reason"] == nil assert summary["total"] == 0 assert summary["completed"] == 0 assert summary["failed"] == 0 assert summary["stopped"] == 0 - assert get_state() == %{"state" => "running", "reason" => :null} + assert get_state() == %{"state" => "running", "reason" => nil} assert get_jobs() == [] end @@ -57,11 +57,11 @@ defmodule ReshardBasicTest do end test "toggle global state" do - assert get_state() == %{"state" => "running", "reason" => :null} + assert get_state() == %{"state" => "running", "reason" => nil} put_state_stopped("xyz") assert get_state() == %{"state" => "stopped", "reason" => "xyz"} put_state_running() - assert get_state() == %{"state" => "running", "reason" => :null} + assert get_state() == %{"state" => "running", "reason" => nil} end test "split q=1 db shards on node1 (1 job)", context do diff --git a/test/elixir/test/reshard_changes_feed.exs b/test/elixir/test/reshard_changes_feed_test.exs similarity index 100% rename from test/elixir/test/reshard_changes_feed.exs rename to test/elixir/test/reshard_changes_feed_test.exs diff --git a/test/elixir/test/rewrite_js_test.exs b/test/elixir/test/rewrite_js_test.exs index a3adb3e7d4b..d793e616ab3 100644 --- a/test/elixir/test/rewrite_js_test.exs +++ b/test/elixir/test/rewrite_js_test.exs @@ -209,203 +209,200 @@ defmodule RewriteJSTest do } } - Enum.each( - ["test_rewrite_suite_db", "test_rewrite_suite_db%2Fwith_slashes"], - fn db_name -> - @tag with_random_db: db_name - test "Test basic js rewrites on #{db_name}", context do - db_name = context[:db_name] - - create_doc(db_name, @ddoc) - - docs1 = make_docs(0..9) - bulk_save(db_name, docs1) - - docs2 = [ - %{"a" => 1, "b" => 1, "string" => "doc 1", "type" => "complex"}, - %{"a" => 1, "b" => 2, "string" => "doc 2", "type" => "complex"}, - %{"a" => "test", "b" => %{}, "string" => "doc 3", "type" => "complex"}, - %{ - "a" => "test", - "b" => ["test", "essai"], - "string" => "doc 4", - "type" => "complex" - }, - %{"a" => %{"c" => 1}, "b" => "", "string" => "doc 5", "type" => "complex"} - ] - - bulk_save(db_name, docs2) - - # Test simple rewriting - resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") - assert resp.body == "This is a base64 encoded text" - assert resp.headers["Content-Type"] == "text/plain" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo2") - assert resp.body == "This is a base64 encoded text" - assert resp.headers["Content-Type"] == "text/plain" - - # Test POST, hello update world - resp = - Couch.post("/#{db_name}", body: %{"word" => "plankton", "name" => "Rusty"}).body - - assert resp["ok"] - doc_id = resp["id"] - assert doc_id - - resp = Couch.put("/#{db_name}/_design/test/_rewrite/hello/#{doc_id}") - assert resp.status_code in [201, 202] - assert resp.body == "hello doc" - assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) - - assert Couch.get("/#{db_name}/#{doc_id}").body["world"] == "hello" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome?name=user") - assert resp.body == "Welcome user" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome/user") - assert resp.body == "Welcome user" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome2") - assert resp.body == "Welcome user" - - resp = Couch.put("/#{db_name}/_design/test/_rewrite/welcome3/test") - assert resp.status_code in [201, 202] - assert resp.body == "New World" - assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome3/test") - assert resp.body == "Welcome test" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome4/user") - assert resp.body == "Welcome user" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome5/welcome3") - assert resp.body == "Welcome welcome3" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/basicView") - assert resp.status_code == 200 - assert resp.body["total_rows"] == 9 - - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/FirstKey: [1, 2]/) - - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView2") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 3/) - - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView3") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 4/) - - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView4") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 5/) - - # COUCHDB-1612 - send body rewriting get to post - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/sendBody1") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 5 LineNo: 1/) - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/db/_design/test?meta=true") - assert resp.status_code == 200 - assert resp.body["_id"] == "_design/test" - assert Map.has_key?(resp.body, "_revs_info") - end - - @tag with_random_db: db_name - test "early response on #{db_name}", context do - db_name = context[:db_name] - - ddoc = %{ - _id: "_design/response", - rewrites: """ - function(req){ - status = parseInt(req.query.status); - return {code: status, - body: JSON.stringify({"status": status}), - headers: {'x-foo': 'bar', 'Content-Type': 'application/json'}}; - } - """ + for db_name <- ["test_rewrite_suite_db", "test_rewrite_suite_db%2Fwith_slashes"] do + @tag with_random_db: db_name + test "basic js rewrites on #{db_name}", context do + db_name = context[:db_name] + + create_doc(db_name, @ddoc) + + docs1 = make_docs(0..9) + bulk_save(db_name, docs1) + + docs2 = [ + %{"a" => 1, "b" => 1, "string" => "doc 1", "type" => "complex"}, + %{"a" => 1, "b" => 2, "string" => "doc 2", "type" => "complex"}, + %{"a" => "test", "b" => %{}, "string" => "doc 3", "type" => "complex"}, + %{ + "a" => "test", + "b" => ["test", "essai"], + "string" => "doc 4", + "type" => "complex" + }, + %{"a" => %{"c" => 1}, "b" => "", "string" => "doc 5", "type" => "complex"} + ] + + bulk_save(db_name, docs2) + + # Test simple rewriting + resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") + assert resp.body == "This is a base64 encoded text" + assert resp.headers["Content-Type"] == "text/plain" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo2") + assert resp.body == "This is a base64 encoded text" + assert resp.headers["Content-Type"] == "text/plain" + + # Test POST, hello update world + resp = + Couch.post("/#{db_name}", body: %{"word" => "plankton", "name" => "Rusty"}).body + + assert resp["ok"] + doc_id = resp["id"] + assert doc_id + + resp = Couch.put("/#{db_name}/_design/test/_rewrite/hello/#{doc_id}") + assert resp.status_code in [201, 202] + assert resp.body == "hello doc" + assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + + assert Couch.get("/#{db_name}/#{doc_id}").body["world"] == "hello" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome?name=user") + assert resp.body == "Welcome user" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome/user") + assert resp.body == "Welcome user" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome2") + assert resp.body == "Welcome user" + + resp = Couch.put("/#{db_name}/_design/test/_rewrite/welcome3/test") + assert resp.status_code in [201, 202] + assert resp.body == "New World" + assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome3/test") + assert resp.body == "Welcome test" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome4/user") + assert resp.body == "Welcome user" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome5/welcome3") + assert resp.body == "Welcome welcome3" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/basicView") + assert resp.status_code == 200 + assert resp.body["total_rows"] == 9 + + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/FirstKey: [1, 2]/) + + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView2") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 3/) + + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView3") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 4/) + + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView4") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 5/) + + # COUCHDB-1612 - send body rewriting get to post + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/sendBody1") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 5 LineNo: 1/) + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/db/_design/test?meta=true") + assert resp.status_code == 200 + assert resp.body["_id"] == "_design/test" + assert Map.has_key?(resp.body, "_revs_info") + end + + @tag with_random_db: db_name + test "early response on #{db_name}", context do + db_name = context[:db_name] + + ddoc = %{ + _id: "_design/response", + rewrites: """ + function(req){ + status = parseInt(req.query.status); + return {code: status, + body: JSON.stringify({"status": status}), + headers: {'x-foo': 'bar', 'Content-Type': 'application/json'}}; } + """ + } - create_doc(db_name, ddoc) + create_doc(db_name, ddoc) - resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=200") - assert resp.status_code == 200 - assert resp.headers["x-foo"] == "bar" - assert resp.body["status"] == 200 + resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=200") + assert resp.status_code == 200 + assert resp.headers["x-foo"] == "bar" + assert resp.body["status"] == 200 - resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=451") - assert resp.status_code == 451 - assert resp.headers["Content-Type"] == "application/json" + resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=451") + assert resp.status_code == 451 + assert resp.headers["Content-Type"] == "application/json" - resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=500") - assert resp.status_code == 500 - end + resp = Couch.get("/#{db_name}/_design/response/_rewrite?status=500") + assert resp.status_code == 500 + end - @tag with_random_db: db_name - test "path relative to server on #{db_name}", context do - db_name = context[:db_name] + @tag with_random_db: db_name + test "path relative to server on #{db_name}", context do + db_name = context[:db_name] - ddoc = %{ - _id: "_design/relative", - rewrites: """ - function(req){ - return '../../../_uuids' - } - """ + ddoc = %{ + _id: "_design/relative", + rewrites: """ + function(req){ + return '../../../_uuids' } + """ + } - create_doc(db_name, ddoc) - resp = Couch.get("/#{db_name}/_design/relative/_rewrite/uuids") - assert resp.status_code == 200 - assert length(resp.body["uuids"]) == 1 - end - - @tag with_random_db: db_name - test "loop on #{db_name}", context do - db_name = context[:db_name] - - ddoc_loop = %{ - _id: "_design/loop", - rewrites: """ - function(req) { - return '_rewrite/loop'; - } - """ + create_doc(db_name, ddoc) + resp = Couch.get("/#{db_name}/_design/relative/_rewrite/uuids") + assert resp.status_code == 200 + assert length(resp.body["uuids"]) == 1 + end + + @tag with_random_db: db_name + test "loop on #{db_name}", context do + db_name = context[:db_name] + + ddoc_loop = %{ + _id: "_design/loop", + rewrites: """ + function(req) { + return '_rewrite/loop'; } + """ + } - create_doc(db_name, ddoc_loop) - resp = Couch.get("/#{db_name}/_design/loop/_rewrite/loop") - assert resp.status_code == 400 - end + create_doc(db_name, ddoc_loop) + resp = Couch.get("/#{db_name}/_design/loop/_rewrite/loop") + assert resp.status_code == 400 + end - @tag with_random_db: db_name - test "requests with body preserve the query string rewrite on #{db_name}", - context do - db_name = context[:db_name] + @tag with_random_db: db_name + test "requests with body preserve the query string rewrite on #{db_name}", + context do + db_name = context[:db_name] - ddoc_qs = %{ - _id: "_design/qs", - rewrites: - "function (r) { return {path: '../../_changes', query: {'filter': '_doc_ids'}};};" - } + ddoc_qs = %{ + _id: "_design/qs", + rewrites: + "function (r) { return {path: '../../_changes', query: {'filter': '_doc_ids'}};};" + } - create_doc(db_name, ddoc_qs) - create_doc(db_name, %{_id: "qs1"}) - create_doc(db_name, %{_id: "qs2"}) + create_doc(db_name, ddoc_qs) + create_doc(db_name, %{_id: "qs1"}) + create_doc(db_name, %{_id: "qs2"}) - resp = - Couch.post("/#{db_name}/_design/qs/_rewrite", - body: %{doc_ids: ["qs2"]} - ) + resp = + Couch.post("/#{db_name}/_design/qs/_rewrite", + body: %{doc_ids: ["qs2"]} + ) - assert resp.status_code == 200 - assert length(resp.body["results"]) == 1 - assert Enum.at(resp.body["results"], 0)["id"] == "qs2" - end + assert resp.status_code == 200 + assert length(resp.body["results"]) == 1 + assert Enum.at(resp.body["results"], 0)["id"] == "qs2" end - ) + end end diff --git a/test/elixir/test/rewrite_test.exs b/test/elixir/test/rewrite_test.exs index e23d6360909..4a13803c688 100644 --- a/test/elixir/test/rewrite_test.exs +++ b/test/elixir/test/rewrite_test.exs @@ -8,519 +8,522 @@ defmodule RewriteTest do This is a port of the rewrite.js suite """ - Enum.each( - ["test_rewrite_suite_db", "test_rewrite_suite_db%2Fwith_slashes"], - fn db_name -> - @tag with_random_db: db_name - @tag config: [ - {"httpd", "authentication_handlers", - "{couch_httpd_auth, special_test_authentication_handler}"}, - {"chttpd", "WWW-Authenticate", "X-Couch-Test-Auth"} - ] - test "Test basic rewrites on #{db_name}", context do - db_name = context[:db_name] - - ddoc = ~S""" - { - "_id": "_design/test", - "language": "javascript", - "_attachments": { - "foo.txt": { - "content_type":"text/plain", - "data": "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=" + for db_name <- ["test_rewrite_suite_db", "test_rewrite_suite_db%2Fwith_slashes"] do + @tag with_random_db: db_name + @tag config: [ + {"httpd", "authentication_handlers", + "{couch_httpd_auth, special_test_authentication_handler}"}, + {"chttpd", "WWW-Authenticate", "X-Couch-Test-Auth"} + ] + test "basic rewrites on #{db_name}", context do + db_name = context[:db_name] + + ddoc = ~S""" + { + "_id": "_design/test", + "language": "javascript", + "_attachments": { + "foo.txt": { + "content_type":"text/plain", + "data": "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=" + } + }, + "rewrites": [ + { + "from": "foo", + "to": "foo.txt" + }, + { + "from": "foo2", + "to": "foo.txt", + "method": "GET" + }, + { + "from": "hello/:id", + "to": "_update/hello/:id", + "method": "PUT" + }, + { + "from": "/welcome", + "to": "_show/welcome" + }, + { + "from": "/welcome/:name", + "to": "_show/welcome", + "query": { + "name": ":name" } }, - "rewrites": [ - { - "from": "foo", - "to": "foo.txt" - }, - { - "from": "foo2", - "to": "foo.txt", - "method": "GET" - }, - { - "from": "hello/:id", - "to": "_update/hello/:id", - "method": "PUT" - }, - { - "from": "/welcome", - "to": "_show/welcome" - }, - { - "from": "/welcome/:name", - "to": "_show/welcome", - "query": { - "name": ":name" - } - }, - { - "from": "/welcome2", - "to": "_show/welcome", - "query": { - "name": "user" - } - }, - { - "from": "/welcome3/:name", - "to": "_update/welcome2/:name", - "method": "PUT" - }, - { - "from": "/welcome3/:name", - "to": "_show/welcome2/:name", - "method": "GET" - }, - { - "from": "/welcome4/*", - "to" : "_show/welcome3", - "query": { - "name": "*" - } - }, - { - "from": "/welcome5/*", - "to" : "_show/*", - "query": { - "name": "*" - } - }, - { - "from": "basicView", - "to": "_view/basicView" - }, - { - "from": "simpleForm/basicView", - "to": "_list/simpleForm/basicView" - }, - { - "from": "simpleForm/basicViewFixed", - "to": "_list/simpleForm/basicView", - "query": { - "startkey": 3, - "endkey": 8 - } - }, - { - "from": "simpleForm/basicViewPath/:start/:end", - "to": "_list/simpleForm/basicView", - "query": { - "startkey": ":start", - "endkey": ":end" - }, - "formats": { - "start": "int", - "end": "int" - } - }, - { - "from": "simpleForm/complexView", - "to": "_list/simpleForm/complexView", - "query": { - "key": [1, 2] - } - }, - { - "from": "simpleForm/complexView2", - "to": "_list/simpleForm/complexView", - "query": { - "key": ["test", {}] - } - }, - { - "from": "simpleForm/complexView3", - "to": "_list/simpleForm/complexView", - "query": { - "key": ["test", ["test", "essai"]] - } - }, - { - "from": "simpleForm/complexView4", - "to": "_list/simpleForm/complexView2", - "query": { - "key": {"c": 1} - } + { + "from": "/welcome2", + "to": "_show/welcome", + "query": { + "name": "user" + } + }, + { + "from": "/welcome3/:name", + "to": "_update/welcome2/:name", + "method": "PUT" + }, + { + "from": "/welcome3/:name", + "to": "_show/welcome2/:name", + "method": "GET" + }, + { + "from": "/welcome4/*", + "to" : "_show/welcome3", + "query": { + "name": "*" + } + }, + { + "from": "/welcome5/*", + "to" : "_show/*", + "query": { + "name": "*" + } + }, + { + "from": "basicView", + "to": "_view/basicView" + }, + { + "from": "simpleForm/basicView", + "to": "_list/simpleForm/basicView" + }, + { + "from": "simpleForm/basicViewFixed", + "to": "_list/simpleForm/basicView", + "query": { + "startkey": 3, + "endkey": 8 + } + }, + { + "from": "simpleForm/basicViewPath/:start/:end", + "to": "_list/simpleForm/basicView", + "query": { + "startkey": ":start", + "endkey": ":end" }, - { - "from": "simpleForm/complexView5/:a/:b", - "to": "_list/simpleForm/complexView3", - "query": { - "key": [":a", ":b"] - } + "formats": { + "start": "int", + "end": "int" + } + }, + { + "from": "simpleForm/complexView", + "to": "_list/simpleForm/complexView", + "query": { + "key": [1, 2] + } + }, + { + "from": "simpleForm/complexView2", + "to": "_list/simpleForm/complexView", + "query": { + "key": ["test", {}] + } + }, + { + "from": "simpleForm/complexView3", + "to": "_list/simpleForm/complexView", + "query": { + "key": ["test", ["test", "essai"]] + } + }, + { + "from": "simpleForm/complexView4", + "to": "_list/simpleForm/complexView2", + "query": { + "key": {"c": 1} + } + }, + { + "from": "simpleForm/complexView5/:a/:b", + "to": "_list/simpleForm/complexView3", + "query": { + "key": [":a", ":b"] + } + }, + { + "from": "simpleForm/complexView6", + "to": "_list/simpleForm/complexView3", + "query": { + "key": [":a", ":b"] + } + }, + { + "from": "simpleForm/complexView7/:a/:b", + "to": "_view/complexView3", + "query": { + "key": [":a", ":b"], + "include_docs": ":doc" }, - { - "from": "simpleForm/complexView6", - "to": "_list/simpleForm/complexView3", - "query": { - "key": [":a", ":b"] + "format": { + "doc": "bool" + } + + }, + { + "from": "/", + "to": "_view/basicView" + }, + { + "from": "/db/*", + "to": "../../*" + } + ], + "lists": { + "simpleForm": "function(head, req) { + log(\"simpleForm\"); + send(\"
    \"); + var row, row_number = 0, prevKey, firstKey = null; + while (row = getRow()) { + row_number += 1; + if (!firstKey) firstKey = row.key; + prevKey = row.key; + send(\"\\n
  • Key: \"+row.key + +\" Value: \"+row.value + +\" LineNo: \"+row_number+\"
    • \"); + } + return \"

FirstKey: \"+ firstKey + \" LastKey: \"+ prevKey+\"

\"; + }" + }, + "shows": { + "welcome": "(function(doc,req) { + return \"Welcome \" + req.query[\"name\"]; + })", + "welcome2": "(function(doc, req) { + return \"Welcome \" + doc.name; + })", + "welcome3": "(function(doc,req) { + return \"Welcome \" + req.query[\"name\"]; + })" + }, + "updates": { + "hello" : "(function(doc, req) { + if (!doc) { + if (req.id) { + return [{ + _id : req.id + }, \"New World\"] } - }, - { - "from": "simpleForm/complexView7/:a/:b", - "to": "_view/complexView3", - "query": { - "key": [":a", ":b"], - "include_docs": ":doc" - }, - "format": { - "doc": "bool" + return [null, \"Empty World\"]; + } + doc.world = \"hello\"; + doc.edited_by = req.userCtx; + return [doc, \"hello doc\"]; + })", + "welcome2": "(function(doc, req) { + if (!doc) { + if (req.id) { + return [{ + _id: req.id, + name: req.id + }, \"New World\"] } - - }, - { - "from": "/", - "to": "_view/basicView" - }, - { - "from": "/db/*", - "to": "../../*" + return [null, \"Empty World\"]; } - ], - "lists": { - "simpleForm": "function(head, req) { - log(\"simpleForm\"); - send(\"
    \"); - var row, row_number = 0, prevKey, firstKey = null; - while (row = getRow()) { - row_number += 1; - if (!firstKey) firstKey = row.key; - prevKey = row.key; - send(\"\\n
  • Key: \"+row.key - +\" Value: \"+row.value - +\" LineNo: \"+row_number+\"
    • \"); + return [doc, \"hello doc\"]; + })" + }, + "views" : { + "basicView" : { + "map" : "(function(doc) { + if (doc.integer) { + emit(doc.integer, doc.string); } - return \"

FirstKey: \"+ firstKey + \" LastKey: \"+ prevKey+\"

\"; - }" - }, - "shows": { - "welcome": "(function(doc,req) { - return \"Welcome \" + req.query[\"name\"]; - })", - "welcome2": "(function(doc, req) { - return \"Welcome \" + doc.name; - })", - "welcome3": "(function(doc,req) { - return \"Welcome \" + req.query[\"name\"]; + })" }, - "updates": { - "hello" : "(function(doc, req) { - if (!doc) { - if (req.id) { - return [{ - _id : req.id - }, \"New World\"] - } - return [null, \"Empty World\"]; + "complexView": { + "map": "(function(doc) { + if (doc.type == \"complex\") { + emit([doc.a, doc.b], doc.string); } - doc.world = \"hello\"; - doc.edited_by = req.userCtx; - return [doc, \"hello doc\"]; - })", - "welcome2": "(function(doc, req) { - if (!doc) { - if (req.id) { - return [{ - _id: req.id, - name: req.id - }, \"New World\"] - } - return [null, \"Empty World\"]; + })" + }, + "complexView2": { + "map": "(function(doc) { + if (doc.type == \"complex\") { + emit(doc.a, doc.string); } - return [doc, \"hello doc\"]; })" }, - "views" : { - "basicView" : { - "map" : "(function(doc) { - if (doc.integer) { - emit(doc.integer, doc.string); - } - - })" - }, - "complexView": { - "map": "(function(doc) { - if (doc.type == \"complex\") { - emit([doc.a, doc.b], doc.string); - } - })" - }, - "complexView2": { - "map": "(function(doc) { - if (doc.type == \"complex\") { - emit(doc.a, doc.string); - } - })" - }, - "complexView3": { - "map": "(function(doc) { - if (doc.type == \"complex\") { - emit(doc.b, doc.string); - } - })" - } + "complexView3": { + "map": "(function(doc) { + if (doc.type == \"complex\") { + emit(doc.b, doc.string); + } + })" } } - """ - - ddoc = String.replace(ddoc, ~r/[\r\n]+/, "") - - docs1 = make_docs(0..9) - - docs2 = [ - %{"a" => 1, "b" => 1, "string" => "doc 1", "type" => "complex"}, - %{"a" => 1, "b" => 2, "string" => "doc 2", "type" => "complex"}, - %{"a" => "test", "b" => %{}, "string" => "doc 3", "type" => "complex"}, - %{ - "a" => "test", - "b" => ["test", "essai"], - "string" => "doc 4", - "type" => "complex" - }, - %{"a" => %{"c" => 1}, "b" => "", "string" => "doc 5", "type" => "complex"} - ] - - assert Couch.put("/#{db_name}/_design/test", body: ddoc).body["ok"] - - assert Couch.post( - "/#{db_name}/_bulk_docs", - body: %{:docs => docs1}, - query: %{w: 3} - ).status_code in [201, 202] - - assert Couch.post( - "/#{db_name}/_bulk_docs", - body: %{:docs => docs2}, - query: %{w: 3} - ).status_code in [201, 202] - - # Test simple rewriting - resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") - assert resp.body == "This is a base64 encoded text" - assert resp.headers["Content-Type"] == "text/plain" - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo2") - assert resp.body == "This is a base64 encoded text" - assert resp.headers["Content-Type"] == "text/plain" - - # Test POST, hello update world - resp = - Couch.post("/#{db_name}", body: %{"word" => "plankton", "name" => "Rusty"}).body - - assert resp["ok"] - doc_id = resp["id"] - assert doc_id - - resp = Couch.put("/#{db_name}/_design/test/_rewrite/hello/#{doc_id}") - assert resp.status_code in [201, 202] - assert resp.body == "hello doc" - assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + } + """ + + ddoc = String.replace(ddoc, ~r/[\r\n]+/, "") + + docs1 = make_docs(0..9) + + docs2 = [ + %{"a" => 1, "b" => 1, "string" => "doc 1", "type" => "complex"}, + %{"a" => 1, "b" => 2, "string" => "doc 2", "type" => "complex"}, + %{"a" => "test", "b" => %{}, "string" => "doc 3", "type" => "complex"}, + %{ + "a" => "test", + "b" => ["test", "essai"], + "string" => "doc 4", + "type" => "complex" + }, + %{"a" => %{"c" => 1}, "b" => "", "string" => "doc 5", "type" => "complex"} + ] + + assert Couch.put("/#{db_name}/_design/test", body: ddoc).body["ok"] + + assert Couch.post( + "/#{db_name}/_bulk_docs", + body: %{:docs => docs1}, + query: %{w: 3} + ).status_code in [201, 202] + + assert Couch.post( + "/#{db_name}/_bulk_docs", + body: %{:docs => docs2}, + query: %{w: 3} + ).status_code in [201, 202] + + # Test simple rewriting + resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") + assert resp.body == "This is a base64 encoded text" + assert resp.headers["Content-Type"] == "text/plain" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo2") + assert resp.body == "This is a base64 encoded text" + assert resp.headers["Content-Type"] == "text/plain" + + # Test POST, hello update world + resp = + Couch.post("/#{db_name}", body: %{"word" => "plankton", "name" => "Rusty"}).body + + assert resp["ok"] + doc_id = resp["id"] + assert doc_id + + resp = Couch.put("/#{db_name}/_design/test/_rewrite/hello/#{doc_id}") + assert resp.status_code in [201, 202] + assert resp.body == "hello doc" + assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + + assert Couch.get("/#{db_name}/#{doc_id}").body["world"] == "hello" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome?name=user") + assert resp.body == "Welcome user" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome/user") + assert resp.body == "Welcome user" + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome2") + assert resp.body == "Welcome user" + + resp = Couch.put("/#{db_name}/_design/test/_rewrite/welcome3/test") + assert resp.status_code in [201, 202] + assert resp.body == "New World" + assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome3/test") + assert resp.body == "Welcome test" - assert Couch.get("/#{db_name}/#{doc_id}").body["world"] == "hello" + # TODO: port the two "bugged" tests from rewrite.js + + resp = Couch.get("/#{db_name}/_design/test/_rewrite/basicView") + assert resp.status_code == 200 + assert resp.body["total_rows"] == 9 + + resp = Couch.get("/#{db_name}/_design/test/_rewrite") + assert resp.status_code == 200 + assert resp.body["total_rows"] == 9 + + resp = + Rawresp.get( + "/#{db_name}/_design/test/_rewrite/simpleForm/basicView?startkey=3&endkey=8" + ) + + assert resp.status_code == 200 + assert not String.match?(resp.body, ~r/Key: 1/) + assert String.match?(resp.body, ~r/FirstKey: 3/) + assert String.match?(resp.body, ~r/LastKey: 8/) + + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/basicViewFixed") + assert resp.status_code == 200 + assert not String.match?(resp.body, ~r/Key: 1/) + assert String.match?(resp.body, ~r/FirstKey: 3/) + assert String.match?(resp.body, ~r/LastKey: 8/) + + resp = + Rawresp.get( + "/#{db_name}/_design/test/_rewrite/simpleForm/basicViewFixed?startkey=4" + ) + + assert resp.status_code == 200 + assert not String.match?(resp.body, ~r/Key: 1/) + assert String.match?(resp.body, ~r/FirstKey: 3/) + assert String.match?(resp.body, ~r/LastKey: 8/) + + resp = + Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/basicViewPath/3/8") + + assert resp.status_code == 200 + assert not String.match?(resp.body, ~r/Key: 1/) + assert String.match?(resp.body, ~r/FirstKey: 3/) + assert String.match?(resp.body, ~r/LastKey: 8/) - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome?name=user") - assert resp.body == "Welcome user" + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/FirstKey: [1, 2]/) - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome/user") - assert resp.body == "Welcome user" + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView2") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 3/) - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome2") - assert resp.body == "Welcome user" + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView3") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 4/) - resp = Couch.put("/#{db_name}/_design/test/_rewrite/welcome3/test") - assert resp.status_code in [201, 202] - assert resp.body == "New World" - assert String.match?(resp.headers["Content-Type"], ~r/charset=utf-8/) + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView4") + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 5/) - resp = Couch.get("/#{db_name}/_design/test/_rewrite/welcome3/test") - assert resp.body == "Welcome test" + resp = + Rawresp.get( + "/#{db_name}/_design/test/_rewrite/simpleForm/complexView5/test/essai" + ) - # TODO: port the two "bugged" tests from rewrite.js - - resp = Couch.get("/#{db_name}/_design/test/_rewrite/basicView") - assert resp.status_code == 200 - assert resp.body["total_rows"] == 9 - - resp = Couch.get("/#{db_name}/_design/test/_rewrite") - assert resp.status_code == 200 - assert resp.body["total_rows"] == 9 + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 4/) - resp = - Rawresp.get( - "/#{db_name}/_design/test/_rewrite/simpleForm/basicView?startkey=3&endkey=8" - ) + resp = + Rawresp.get( + "/#{db_name}/_design/test/_rewrite/simpleForm/complexView6?a=test&b=essai" + ) - assert resp.status_code == 200 - assert not String.match?(resp.body, ~r/Key: 1/) - assert String.match?(resp.body, ~r/FirstKey: 3/) - assert String.match?(resp.body, ~r/LastKey: 8/) + assert resp.status_code == 200 + assert String.match?(resp.body, ~r/Value: doc 4/) - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/basicViewFixed") - assert resp.status_code == 200 - assert not String.match?(resp.body, ~r/Key: 1/) - assert String.match?(resp.body, ~r/FirstKey: 3/) - assert String.match?(resp.body, ~r/LastKey: 8/) + resp = + Rawresp.get( + "/#{db_name}/_design/test/_rewrite/simpleForm/complexView7/test/essai?doc=true" + ) - resp = - Rawresp.get( - "/#{db_name}/_design/test/_rewrite/simpleForm/basicViewFixed?startkey=4" - ) + assert resp.status_code == 200 - assert resp.status_code == 200 - assert not String.match?(resp.body, ~r/Key: 1/) - assert String.match?(resp.body, ~r/FirstKey: 3/) - assert String.match?(resp.body, ~r/LastKey: 8/) + result = + resp.body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps, :use_nil]) - resp = - Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/basicViewPath/3/8") + first_row = Enum.at(result["rows"], 0) + assert Map.has_key?(first_row, "doc") - assert resp.status_code == 200 - assert not String.match?(resp.body, ~r/Key: 1/) - assert String.match?(resp.body, ~r/FirstKey: 3/) - assert String.match?(resp.body, ~r/LastKey: 8/) + # COUCHDB-2031 - path normalization versus qs params + resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/db/_design/test?meta=true") + assert resp.status_code == 200 - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/FirstKey: [1, 2]/) + result = + resp.body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps, :use_nil]) - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView2") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 3/) + assert result["_id"] == "_design/test" + assert Map.has_key?(result, "_revs_info") - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView3") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 4/) + ddoc2 = %{ + _id: "_design/test2", + rewrites: [ + %{ + from: "uuids", + to: "../../../_uuids" + } + ] + } - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/simpleForm/complexView4") - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 5/) + create_doc(db_name, ddoc2) + resp = Couch.get("/#{db_name}/_design/test2/_rewrite/uuids") + assert resp.status_code == 500 + assert resp.body["error"] == "insecure_rewrite_rule" + end - resp = - Rawresp.get( - "/#{db_name}/_design/test/_rewrite/simpleForm/complexView5/test/essai" - ) + @tag with_random_db: db_name + @tag config: [ + {"chttpd", "secure_rewrites", "false"} + ] + test "path relative to server on #{db_name}", context do + db_name = context[:db_name] - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 4/) + ddoc = %{ + _id: "_design/test2", + rewrites: [ + %{ + from: "uuids", + to: "../../../_uuids" + } + ] + } - resp = - Rawresp.get( - "/#{db_name}/_design/test/_rewrite/simpleForm/complexView6?a=test&b=essai" - ) + create_doc(db_name, ddoc) - assert resp.status_code == 200 - assert String.match?(resp.body, ~r/Value: doc 4/) + resp = Couch.get("/#{db_name}/_design/test2/_rewrite/uuids") + assert resp.status_code == 200 + assert length(resp.body["uuids"]) == 1 + end - resp = - Rawresp.get( - "/#{db_name}/_design/test/_rewrite/simpleForm/complexView7/test/essai?doc=true" - ) + @tag with_random_db: db_name + @tag config: [ + {"chttpd", "rewrite_limit", "2"} + ] + test "loop detection on #{db_name}", context do + db_name = context[:db_name] - assert resp.status_code == 200 - result = resp.body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps]) - first_row = Enum.at(result["rows"], 0) - assert Map.has_key?(first_row, "doc") + ddoc_loop = %{ + _id: "_design/loop", + rewrites: [%{from: "loop", to: "_rewrite/loop"}] + } - # COUCHDB-2031 - path normalization versus qs params - resp = Rawresp.get("/#{db_name}/_design/test/_rewrite/db/_design/test?meta=true") - assert resp.status_code == 200 - result = resp.body |> IO.iodata_to_binary() |> :jiffy.decode([:return_maps]) - assert result["_id"] == "_design/test" - assert Map.has_key?(result, "_revs_info") - - ddoc2 = %{ - _id: "_design/test2", - rewrites: [ - %{ - from: "uuids", - to: "../../../_uuids" - } - ] - } + create_doc(db_name, ddoc_loop) - create_doc(db_name, ddoc2) - resp = Couch.get("/#{db_name}/_design/test2/_rewrite/uuids") - assert resp.status_code == 500 - assert resp.body["error"] == "insecure_rewrite_rule" - end + resp = Couch.get("/#{db_name}/_design/loop/_rewrite/loop") + assert resp.status_code == 400 + end - @tag with_random_db: db_name - @tag config: [ - {"chttpd", "secure_rewrites", "false"} - ] - test "path relative to server on #{db_name}", context do - db_name = context[:db_name] - - ddoc = %{ - _id: "_design/test2", - rewrites: [ - %{ - from: "uuids", - to: "../../../_uuids" - } - ] - } + @tag with_random_db: db_name + @tag config: [ + {"chttpd", "rewrite_limit", "2"}, + {"chttpd", "secure_rewrites", "false"} + ] + test "serial execution is not spuriously counted as loop on #{db_name}", context do + db_name = context[:db_name] + + ddoc = %{ + _id: "_design/test", + language: "javascript", + _attachments: %{ + "foo.txt": %{ + content_type: "text/plain", + data: "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=" + } + }, + rewrites: [ + %{ + from: "foo", + to: "foo.txt" + } + ] + } - create_doc(db_name, ddoc) + create_doc(db_name, ddoc) - resp = Couch.get("/#{db_name}/_design/test2/_rewrite/uuids") + for _i <- 0..4 do + resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") assert resp.status_code == 200 - assert length(resp.body["uuids"]) == 1 - end - - @tag with_random_db: db_name - @tag config: [ - {"chttpd", "rewrite_limit", "2"} - ] - test "loop detection on #{db_name}", context do - db_name = context[:db_name] - - ddoc_loop = %{ - _id: "_design/loop", - rewrites: [%{from: "loop", to: "_rewrite/loop"}] - } - - create_doc(db_name, ddoc_loop) - - resp = Couch.get("/#{db_name}/_design/loop/_rewrite/loop") - assert resp.status_code == 400 - end - - @tag with_random_db: db_name - @tag config: [ - {"chttpd", "rewrite_limit", "2"}, - {"chttpd", "secure_rewrites", "false"} - ] - test "serial execution is not spuriously counted as loop on #{db_name}", context do - db_name = context[:db_name] - - ddoc = %{ - _id: "_design/test", - language: "javascript", - _attachments: %{ - "foo.txt": %{ - content_type: "text/plain", - data: "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ=" - } - }, - rewrites: [ - %{ - from: "foo", - to: "foo.txt" - } - ] - } - - create_doc(db_name, ddoc) - - for _i <- 0..4 do - resp = Couch.get("/#{db_name}/_design/test/_rewrite/foo") - assert resp.status_code == 200 - end end end - ) + end end diff --git a/test/elixir/test/support/friend_docs.ex b/test/elixir/test/support/friend_docs.ex new file mode 100644 index 00000000000..0e36af80633 --- /dev/null +++ b/test/elixir/test/support/friend_docs.ex @@ -0,0 +1,286 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule FriendDocs do + @moduledoc """ + Generated with http://www.json-generator.com/ + + With this pattern: + + [ + '{{repeat(15)}}', + { + _id: '{{index()}}', + name: { + first: '{{firstName()}}', + last: '{{surname()}}' + }, + friends: [ + '{{repeat(3)}}', + { + id: '{{index()}}', + name: { + first: '{{firstName()}}', + last: '{{surname()}}' + }, + type: '{{random("personal", "work")}}' + } + ] + } + ] + """ + + @docs [ + %{ + "_id" => "54a43171d37ae5e81bff5ae0", + "user_id" => 0, + "name" => %{"first" => "Ochoa", "last" => "Fox"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Sherman", "last" => "Davidson"}, + "type" => "personal", + }, + %{ + "id" => 1, + "name" => %{"first" => "Vargas", "last" => "Mendez"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Sheppard", "last" => "Cotton"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171958485dc32917c50", + "user_id" => 1, + "name" => %{"first" => "Sheppard", "last" => "Cotton"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Ochoa", "last" => "Fox"}, "type" => "work"}, + %{ + "id" => 1, + "name" => %{"first" => "Vargas", "last" => "Mendez"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Kendra", "last" => "Burns"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a431711cf025ba74bea899", + "user_id" => 2, + "name" => %{"first" => "Hunter", "last" => "Wells"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Estes", "last" => "Fischer"}, "type" => "work"}, + %{ + "id" => 1, + "name" => %{"first" => "Farrell", "last" => "Maddox"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Kendra", "last" => "Burns"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a4317151a70a9881ac28a4", + "user_id" => 3, + "name" => %{"first" => "Millicent", "last" => "Guy"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Luella", "last" => "Mendoza"}, "type" => "work"}, + %{ + "id" => 1, + "name" => %{"first" => "Melanie", "last" => "Foster"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Hopkins", "last" => "Scott"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171d946b78703a0e076", + "user_id" => 4, + "name" => %{"first" => "Elisabeth", "last" => "Brady"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Sofia", "last" => "Workman"}, "type" => "work"}, + %{"id" => 1, "name" => %{"first" => "Alisha", "last" => "Reilly"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Ochoa", "last" => "Burch"}, "type" => "personal"}, + ], + }, + %{ + "_id" => "54a4317118abd7f1992464ee", + "user_id" => 5, + "name" => %{"first" => "Pollard", "last" => "French"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Hollie", "last" => "Juarez"}, + "type" => "personal", + }, + %{"id" => 1, "name" => %{"first" => "Nelda", "last" => "Newton"}, "type" => "personal"}, + %{"id" => 2, "name" => %{"first" => "Yang", "last" => "Pace"}, "type" => "personal"}, + ], + }, + %{ + "_id" => "54a43171f139e63d6579121e", + "user_id" => 6, + "name" => %{"first" => "Acevedo", "last" => "Morales"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Payne", "last" => "Berry"}, "type" => "personal"}, + %{ + "id" => 1, + "name" => %{"first" => "Rene", "last" => "Valenzuela"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Dora", "last" => "Gallegos"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a431719783cef80876dde8", + "user_id" => 7, + "name" => %{"first" => "Cervantes", "last" => "Marquez"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Maxwell", "last" => "Norman"}, + "type" => "personal", + }, + %{"id" => 1, "name" => %{"first" => "Shields", "last" => "Bass"}, "type" => "personal"}, + %{"id" => 2, "name" => %{"first" => "Luz", "last" => "Jacobson"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171ecc7540d1f7aceae", + "user_id" => 8, + "name" => %{"first" => "West", "last" => "Morrow"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Townsend", "last" => "Dixon"}, + "type" => "personal", + }, + %{ + "id" => 1, + "name" => %{"first" => "Callahan", "last" => "Buck"}, + "type" => "personal", + }, + %{ + "id" => 2, + "name" => %{"first" => "Rachel", "last" => "Fletcher"}, + "type" => "personal", + }, + ], + }, + %{ + "_id" => "54a4317113e831f4af041a0a", + "user_id" => 9, + "name" => %{"first" => "Cotton", "last" => "House"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Mckenzie", "last" => "Medina"}, + "type" => "personal", + }, + %{"id" => 1, "name" => %{"first" => "Cecilia", "last" => "Miles"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Guerra", "last" => "Cervantes"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171686eb1f48ebcbe01", + "user_id" => 10, + "name" => %{"first" => "Wright", "last" => "Rivas"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Campos", "last" => "Freeman"}, + "type" => "personal", + }, + %{ + "id" => 1, + "name" => %{"first" => "Christian", "last" => "Ferguson"}, + "type" => "personal", + }, + %{"id" => 2, "name" => %{"first" => "Doreen", "last" => "Wilder"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171a4f3d5638c162f4f", + "user_id" => 11, + "name" => %{"first" => "Lorene", "last" => "Dorsey"}, + "friends" => [ + %{ + "id" => 0, + "name" => %{"first" => "Gibbs", "last" => "Mccarty"}, + "type" => "personal", + }, + %{"id" => 1, "name" => %{"first" => "Neal", "last" => "Franklin"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Kristy", "last" => "Head"}, "type" => "personal"}, + ], + "bestfriends" => ["Wolverine", "Cyclops"], + }, + %{ + "_id" => "54a431719faa420a5b4fbeb0", + "user_id" => 12, + "name" => %{"first" => "Juanita", "last" => "Cook"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Wilkins", "last" => "Chang"}, "type" => "work"}, + %{"id" => 1, "name" => %{"first" => "Haney", "last" => "Rivera"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Lauren", "last" => "Manning"}, "type" => "work"}, + ], + }, + %{ + "_id" => "54a43171e65d35f9ee8c53c0", + "user_id" => 13, + "name" => %{"first" => "Levy", "last" => "Osborn"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Vinson", "last" => "Vargas"}, "type" => "work"}, + %{"id" => 1, "name" => %{"first" => "Felicia", "last" => "Beach"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Nadine", "last" => "Kemp"}, "type" => "work"}, + ], + "results" => [82, 85, 88], + }, + %{ + "_id" => "54a4317132f2c81561833259", + "user_id" => 14, + "name" => %{"first" => "Christina", "last" => "Raymond"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Herrera", "last" => "Walton"}, "type" => "work"}, + %{"id" => 1, "name" => %{"first" => "Hahn", "last" => "Rutledge"}, "type" => "work"}, + %{"id" => 2, "name" => %{"first" => "Stacie", "last" => "Harding"}, "type" => "work"}, + ], + }, + %{ + "_id" => "589f32af493145f890e1b051", + "user_id" => 15, + "name" => %{"first" => "Tanisha", "last" => "Bowers"}, + "friends" => [ + %{"id" => 0, "name" => %{"first" => "Ochoa", "last" => "Pratt"}, "type" => "personal"}, + %{"id" => 1, "name" => %{"first" => "Ochoa", "last" => "Romero"}, "type" => "personal"}, + %{"id" => 2, "name" => %{"first" => "Ochoa", "last" => "Bowman"}, "type" => "work"}, + ], + }, + ] + + def setup(db, index_type \\ "view") do + MangoDatabase.recreate(db) + MangoDatabase.save_docs(db, @docs) + + case index_type do + "view" -> add_view_indexes(db) + "text" -> add_text_indexes(db) + end + + :ok + end + + defp add_view_indexes(_db) do + # TODO: this function is not defined in the Python version of this module? + end + + defp add_text_indexes(db) do + MangoDatabase.create_text_index(db) + end +end diff --git a/test/elixir/test/support/limit_docs.ex b/test/elixir/test/support/limit_docs.ex new file mode 100644 index 00000000000..4ae9a4c3d61 --- /dev/null +++ b/test/elixir/test/support/limit_docs.ex @@ -0,0 +1,112 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule LimitDocs do + @moduledoc false + @docs [ + %{"_id" => "54af50626de419f5109c962f", "user_id" => 0, "age" => 10}, + %{"_id" => "54af50622071121b25402dc3", "user_id" => 1, "age" => 11}, + %{"_id" => "54af50623809e19159a3cdd0", "user_id" => 2, "age" => 12}, + %{"_id" => "54af50629f45a0f49a441d01", "user_id" => 3, "age" => 13}, + %{"_id" => "54af50620f1755c22359a362", "user_id" => 4, "age" => 14}, + %{"_id" => "54af5062dd6f6c689ad2ca23", "user_id" => 5, "age" => 15}, + %{"_id" => "54af50623e89b432be1187b8", "user_id" => 6, "age" => 16}, + %{"_id" => "54af5062932a00270a3b5ab0", "user_id" => 7, "age" => 17}, + %{"_id" => "54af5062df773d69174e3345", "filtered_array" => [1, 2, 3], "age" => 18}, + %{"_id" => "54af50629c1153b9e21e346d", "filtered_array" => [1, 2, 3], "age" => 19}, + %{"_id" => "54af5062dabb7cc4b60e0c95", "user_id" => 10, "age" => 20}, + %{"_id" => "54af5062204996970a4439a2", "user_id" => 11, "age" => 21}, + %{"_id" => "54af50629cea39e8ea52bfac", "user_id" => 12, "age" => 22}, + %{"_id" => "54af50620597c094f75db2a1", "user_id" => 13, "age" => 23}, + %{"_id" => "54af50628d4048de0010723c", "user_id" => 14, "age" => 24}, + %{"_id" => "54af5062f339b6f44f52faf6", "user_id" => 15, "age" => 25}, + %{"_id" => "54af5062a893f17ea4402031", "user_id" => 16, "age" => 26}, + %{"_id" => "54af5062323dbc7077deb60a", "user_id" => 17, "age" => 27}, + %{"_id" => "54af506224db85bd7fcd0243", "filtered_array" => [1, 2, 3], "age" => 28}, + %{"_id" => "54af506255bb551c9cc251bf", "filtered_array" => [1, 2, 3], "age" => 29}, + %{"_id" => "54af50625a97394e07d718a1", "filtered_array" => [1, 2, 3], "age" => 30}, + %{"_id" => "54af506223f51d586b4ef529", "user_id" => 21, "age" => 31}, + %{"_id" => "54af50622740dede7d6117b7", "user_id" => 22, "age" => 32}, + %{"_id" => "54af50624efc87684a52e8fb", "user_id" => 23, "age" => 33}, + %{"_id" => "54af5062f40932760347799c", "user_id" => 24, "age" => 34}, + %{"_id" => "54af5062d9f7361951ac645d", "user_id" => 25, "age" => 35}, + %{"_id" => "54af5062f89aef302b37c3bc", "filtered_array" => [1, 2, 3], "age" => 36}, + %{"_id" => "54af5062498ec905dcb351f8", "filtered_array" => [1, 2, 3], "age" => 37}, + %{"_id" => "54af5062b1d2f2c5a85bdd7e", "user_id" => 28, "age" => 38}, + %{"_id" => "54af50625061029c0dd942b5", "filtered_array" => [1, 2, 3], "age" => 39}, + %{"_id" => "54af50628b0d08a1d23c030a", "user_id" => 30, "age" => 40}, + %{"_id" => "54af506271b6e3119eb31d46", "filtered_array" => [1, 2, 3], "age" => 41}, + %{"_id" => "54af5062b69f46424dfcf3e5", "user_id" => 32, "age" => 42}, + %{"_id" => "54af5062ed00c7dbe4d1bdcf", "user_id" => 33, "age" => 43}, + %{"_id" => "54af5062fb64e45180c9a90d", "user_id" => 34, "age" => 44}, + %{"_id" => "54af5062241c72b067127b09", "user_id" => 35, "age" => 45}, + %{"_id" => "54af50626a467d8b781a6d06", "user_id" => 36, "age" => 46}, + %{"_id" => "54af50620e992d60af03bf86", "filtered_array" => [1, 2, 3], "age" => 47}, + %{"_id" => "54af506254f992aa3c51532f", "user_id" => 38, "age" => 48}, + %{"_id" => "54af5062e99b20f301de39b9", "user_id" => 39, "age" => 49}, + %{"_id" => "54af50624fbade6b11505b5d", "user_id" => 40, "age" => 50}, + %{"_id" => "54af506278ad79b21e807ae4", "user_id" => 41, "age" => 51}, + %{"_id" => "54af5062fc7a1dcb33f31d08", "user_id" => 42, "age" => 52}, + %{"_id" => "54af5062ea2c954c650009cf", "user_id" => 43, "age" => 53}, + %{"_id" => "54af506213576c2f09858266", "user_id" => 44, "age" => 54}, + %{"_id" => "54af50624a05ac34c994b1c0", "user_id" => 45, "age" => 55}, + %{"_id" => "54af50625a624983edf2087e", "user_id" => 46, "age" => 56}, + %{"_id" => "54af50623de488c49d064355", "user_id" => 47, "age" => 57}, + %{"_id" => "54af5062628b5df08661a9d5", "user_id" => 48, "age" => 58}, + %{"_id" => "54af50620c706fc23032ae62", "user_id" => 49, "age" => 59}, + %{"_id" => "54af5062509f1e2371fe1da4", "user_id" => 50, "age" => 60}, + %{"_id" => "54af50625e96b22436791653", "user_id" => 51, "age" => 61}, + %{"_id" => "54af5062a9cb71463bb9577f", "user_id" => 52, "age" => 62}, + %{"_id" => "54af50624fea77a4221a4baf", "user_id" => 53, "age" => 63}, + %{"_id" => "54af5062c63df0a147d2417e", "user_id" => 54, "age" => 64}, + %{"_id" => "54af50623c56d78029316c9f", "user_id" => 55, "age" => 65}, + %{"_id" => "54af5062167f6e13aa0dd014", "user_id" => 56, "age" => 66}, + %{"_id" => "54af50621558abe77797d137", "filtered_array" => [1, 2, 3], "age" => 67}, + %{"_id" => "54af50624d5b36aa7cb5fa77", "user_id" => 58, "age" => 68}, + %{"_id" => "54af50620d79118184ae66bd", "user_id" => 59, "age" => 69}, + %{"_id" => "54af5062d18aafa5c4ca4935", "user_id" => 60, "age" => 71}, + %{"_id" => "54af5062fd22a409649962f4", "filtered_array" => [1, 2, 3], "age" => 72}, + %{"_id" => "54af5062e31045a1908e89f9", "user_id" => 62, "age" => 73}, + %{"_id" => "54af50624c062fcb4c59398b", "user_id" => 63, "age" => 74}, + %{"_id" => "54af506241ec83430a15957f", "user_id" => 64, "age" => 75}, + %{"_id" => "54af506224d0f888ae411101", "user_id" => 65, "age" => 76}, + %{"_id" => "54af506272a971c6cf3ab6b8", "user_id" => 66, "age" => 77}, + %{"_id" => "54af506221e25b485c95355b", "user_id" => 67, "age" => 78}, + %{"_id" => "54af5062800f7f2ca73e9623", "user_id" => 68, "age" => 79}, + %{"_id" => "54af5062bc962da30740534a", "user_id" => 69, "age" => 80}, + %{"_id" => "54af50625102d6e210fc2efd", "filtered_array" => [1, 2, 3], "age" => 81}, + %{"_id" => "54af5062e014b9d039f02c5e", "user_id" => 71, "age" => 82}, + %{"_id" => "54af5062fbd5e801dd217515", "user_id" => 72, "age" => 83}, + %{"_id" => "54af50629971992b658fcb88", "user_id" => 73, "age" => 84}, + %{"_id" => "54af5062607d53416c30bafd", "filtered_array" => [1, 2, 3], "age" => 85}, + ] + + def setup(db, index_type \\ "view") do + MangoDatabase.recreate(db) + MangoDatabase.save_docs(db, @docs) + + case index_type do + "view" -> add_view_indexes(db) + "text" -> add_text_indexes(db) + end + + :ok + end + + defp add_view_indexes(_db) do + # TODO: this function is not defined in the Python version of this module? + end + + defp add_text_indexes(db) do + MangoDatabase.create_text_index(db) + end +end diff --git a/test/elixir/test/support/mango_database.ex b/test/elixir/test/support/mango_database.ex new file mode 100644 index 00000000000..af0788eed1f --- /dev/null +++ b/test/elixir/test/support/mango_database.ex @@ -0,0 +1,203 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule MangoDatabase do + @moduledoc false + def has_text_service() do + resp = Couch.get("/") + "search" in resp.body["features"] + end + + def recreate(db, opts \\ []) do + resp = Couch.get("/#{db}") + if resp.status_code == 200 do + docs = resp.body["doc_count"] + resp.body["doc_del_count"] + if docs > 0 do + delete(db) + create(db, opts) + end + else + create(db, opts) + end + end + + def create(db, opts \\ []) do + partitioned = Keyword.get(opts, :partitioned, false) + Couch.put("/#{db}?partitioned=#{partitioned}") + end + + defp delete(db) do + Couch.delete("/#{db}") + end + + def save_doc(db, doc, opts \\ []) do + MangoDatabase.save_docs(db, [doc], opts) + end + + def save_docs_with_conflicts(db, docs) do + body = %{"docs" => docs, "new_edits" => false} + Couch.post("/#{db}/_bulk_docs", body: body) + end + + # If a certain keyword like sort or field is passed in the options, + # then it is added to the request body. + defp put_if_set(map, key, opts, opts_key) do + if Keyword.has_key?(opts, opts_key) do + Map.put(map, key, opts[opts_key]) + else + map + end + end + + # TODO: make this use batches if necessary + def save_docs(db, docs, opts \\ []) do + query = %{} + |> put_if_set("w", opts, :w) + + result = Couch.post("/#{db}/_bulk_docs", body: %{"docs" => docs}, query: query) + zipped_docs = Enum.zip(docs, result.body) + + # This returns the doc list including _id and _rev values + resp = Enum.map(zipped_docs, fn {doc, result} -> + doc + |> Map.put("_id", result["id"]) + |> Map.put("_rev", result["rev"]) + end) + + # _bulk_docs sometimes returns errors in the body and this is captured here + errors = Enum.filter(result.body, fn r -> + Map.has_key?(r, "error") + end) + if errors == [] do + resp + else + {:error, errors} + end + end + + def create_index(db, fields, options \\ []) do + index = %{ + "fields" => fields, + } + |> put_if_set("selector", options, :selector) + |> put_if_set("partial_filter_selector", options, :partial_filter_selector) + + body = %{ + "index" => index, + "type" => "json", + "w" => 3 + } + |> put_if_set("type", options, :idx_type) + |> put_if_set("name", options, :name) + |> put_if_set("ddoc", options, :ddoc) + + resp = Couch.post("/#{db}/_index", body: body) + if resp.status_code == 200 do + {:ok, resp.body["result"] == "created"} + else + {:error, resp} + end + end + + def create_text_index(db, options \\ []) do + index = %{} + |> put_if_set("default_analyzer", options, :analyzer) + |> put_if_set("default_field", options, :default_field) + |> put_if_set("index_array_lengths", options, :index_array_lengths) + |> put_if_set("selector", options, :selector) + |> put_if_set("partial_filter_selector", options, :partial_filter_selector) + |> put_if_set("fields", options, :fields) + + body = %{ + "index" => index, + "type" => Keyword.get(options, :idx_type, "text"), + "w" => 3 + } + |> put_if_set("name", options, :name) + |> put_if_set("ddoc", options, :ddoc) + + resp = Couch.post("/#{db}/_index", body: body) + + if resp.status_code == 200 do + {:ok, resp.body["result"] == "created"} + else + {:error, resp} + end + end + + def list_indexes(db, opts \\ []) do + limit = Keyword.get(opts, :limit) + skip = Keyword.get(opts, :skip) + query = + [limit: limit, skip: skip] + |> Enum.filter(fn {_k, v} -> not is_nil(v) end) + |> Enum.map_join("&", fn {k, v} -> "#{k}=#{v}" end) + + path = + if query == "" do + "/#{db}/_index" + else + "/#{db}/_index?#{query}" + end + resp = Couch.get(path) + + if resp.status_code == 200 do + {:ok, resp.body["indexes"]} + else + {:error, resp} + end + end + + def find(db, selector, opts \\ []) do + defaults = [ + use_index: nil, + skip: 0, + limit: 25, + r: 1, + conflicts: false, + explain: false, + return_raw: false, + partition: false, + ] + options = Keyword.merge(defaults, opts) + + ppath = + if options[:partition], + do: "_partition/#{options[:partition]}/", + else: "" + + suffix = if options[:explain], do: "_explain", else: "_find" + path = "/#{db}/#{ppath}#{suffix}" + + resp = + Couch.post(path, body: %{ + "selector" => selector, + "use_index" => options[:use_index], + "skip" => options[:skip], + "limit" => options[:limit], + "r" => options[:r], + "conflicts" => options[:conflicts] + } + |> put_if_set("sort", options, :sort) + |> put_if_set("fields", options, :fields) + |> put_if_set("execution_stats", options, :executionStats) + |> put_if_set("allow_fallback", options, :allow_fallback) + |> put_if_set("bookmark", options, :bookmark) + ) + + case {(options[:explain] or options[:return_raw]), resp.status_code} do + {false, 200} -> {:ok, resp.body["docs"]} + {true, 200} -> {:ok, resp.body} + _ -> {:error, resp} + end + end +end diff --git a/test/elixir/test/support/user_docs.ex b/test/elixir/test/support/user_docs.ex new file mode 100644 index 00000000000..52dbca4ae64 --- /dev/null +++ b/test/elixir/test/support/user_docs.ex @@ -0,0 +1,401 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +defmodule UserDocs do + @moduledoc """ + Generated with http://www.json-generator.com/ + + With this pattern: + + [ + '{{repeat(20)}}', + { + _id: '{{guid()}}', + user_id: "{{index()}}", + name: { + first: "{{firstName()}}", + last: "{{surname()}}" + }, + age: "{{integer(18,90)}}", + location: { + state: "{{state()}}", + city: "{{city()}}", + address: { + street: "{{street()}}", + number: "{{integer(10, 10000)}}" + } + }, + company: "{{company()}}", + email: "{{email()}}", + manager: "{{bool()}}", + twitter: function(tags) { + if(this.manager) + return; + return "@" + this.email.split("@")[0]; + }, + favorites: [ + "{{repeat(2,5)}}", + "{{random('C', 'C++', 'Python', 'Ruby', 'Erlang', 'Lisp')}}" + ] + } + ] + """ + + @partitions 3 + + @docs [ + %{ + "_id" => "71562648-6acb-42bc-a182-df6b1f005b09", + "user_id" => 0, + "name" => %{"first" => "Stephanie", "last" => "Kirkland"}, + "age" => 48, + "location" => %{ + "state" => "Nevada", + "city" => "Ronco", + "address" => %{"street" => "Evergreen Avenue", "number" => 347}, + }, + "company" => "Dreamia", + "email" => "stephaniekirkland@dreamia.com", + "manager" => false, + "twitter" => "@stephaniekirkland", + "favorites" => ["Ruby", "C", "Python"], + "test" => [%{"a" => 1}, %{"b" => 2}], + }, + %{ + "_id" => "12a2800c-4fe2-45a8-8d78-c084f4e242a9", + "user_id" => 1, + "name" => %{"first" => "Abbott", "last" => "Watson"}, + "age" => 31, + "location" => %{ + "state" => "Connecticut", + "city" => "Gerber", + "address" => %{"street" => "Huntington Street", "number" => 8987}, + }, + "company" => "Talkola", + "email" => "abbottwatson@talkola.com", + "manager" => false, + "twitter" => "@abbottwatson", + "favorites" => ["Ruby", "Python", "C", %{"Versions" => %{"Alpha" => "Beta"}}], + "test" => [%{"a" => 1, "b" => 2}], + }, + %{ + "_id" => "48ca0455-8bd0-473f-9ae2-459e42e3edd1", + "user_id" => 2, + "name" => %{"first" => "Shelly", "last" => "Ewing"}, + "age" => 42, + "location" => %{ + "state" => "New Mexico", + "city" => "Thornport", + "address" => %{"street" => "Miller Avenue", "number" => 7100}, + }, + "company" => "Zialactic", + "email" => "shellyewing@zialactic.com", + "manager" => true, + "favorites" => ["Lisp", "Python", "Erlang"], + "test_in" => %{"val1" => 1, "val2" => "val2"}, + }, + %{ + "_id" => "0461444c-e60a-457d-a4bb-b8d811853f21", + "user_id" => 3, + "name" => %{"first" => "Madelyn", "last" => "Soto"}, + "age" => 79, + "location" => %{ + "state" => "Utah", + "city" => "Albany", + "address" => %{"street" => "Stockholm Street", "number" => 710}, + }, + "company" => "Tasmania", + "email" => "madelynsoto@tasmania.com", + "manager" => true, + "favorites" => [["Lisp", "Erlang", "Python"], "Erlang", "C", "Erlang"], + "11111" => "number_field", + "22222" => %{"33333" => "nested_number_field"}, + }, + %{ + "_id" => "8e1c90c0-ac18-4832-8081-40d14325bde0", + "user_id" => 4, + "name" => %{"first" => "Nona", "last" => "Horton"}, + "age" => 61, + "location" => %{ + "state" => "Georgia", + "city" => "Corinne", + "address" => %{"street" => "Woodhull Street", "number" => 6845}, + }, + "company" => "Signidyne", + "email" => "nonahorton@signidyne.com", + "manager" => false, + "twitter" => "@nonahorton", + "favorites" => ["Lisp", "C", "Ruby", "Ruby"], + "name.first" => "name dot first", + }, + %{ + "_id" => "a33d5457-741a-4dce-a217-3eab28b24e3e", + "user_id" => 5, + "name" => %{"first" => "Sheri", "last" => "Perkins"}, + "age" => 73, + "location" => %{ + "state" => "Michigan", + "city" => "Nutrioso", + "address" => %{"street" => "Bassett Avenue", "number" => 5648}, + }, + "company" => "Myopium", + "email" => "sheriperkins@myopium.com", + "manager" => true, + "favorites" => ["Lisp", "Lisp"], + }, + %{ + "_id" => "b31dad3f-ae8b-4f86-8327-dfe8770beb27", + "user_id" => 6, + "name" => %{"first" => "Tate", "last" => "Guy"}, + "age" => 47, + "location" => %{ + "state" => "Illinois", + "city" => "Helen", + "address" => %{"street" => "Schenck Court", "number" => 7392}, + }, + "company" => "Prosely", + "email" => "tateguy@prosely.com", + "manager" => true, + "favorites" => ["C", "Lisp", "Ruby", "C"], + }, + %{ + "_id" => "659d0430-b1f4-413a-a6b7-9ea1ef071325", + "user_id" => 7, + "name" => %{"first" => "Jewell", "last" => "Stafford"}, + "age" => 33, + "location" => %{ + "state" => "Iowa", + "city" => "Longbranch", + "address" => %{"street" => "Dodworth Street", "number" => 3949}, + }, + "company" => "Niquent", + "email" => "jewellstafford@niquent.com", + "manager" => true, + "favorites" => ["C", "C", "Ruby", "Ruby", "Erlang"], + "exists_field" => "should_exist1", + "ordered" => nil, + }, + %{ + "_id" => "6c0afcf1-e57e-421d-a03d-0c0717ebf843", + "user_id" => 8, + "name" => %{"first" => "James", "last" => "Mcdaniel"}, + "age" => 68, + "location" => %{ + "state" => "Maine", + "city" => "Craig", + "address" => %{"street" => "Greene Avenue", "number" => 8776}, + }, + "company" => "Globoil", + "email" => "jamesmcdaniel@globoil.com", + "manager" => true, + "favorites" => nil, + "exists_field" => "should_exist2", + "ordered" => false, + }, + %{ + "_id" => "954272af-d5ed-4039-a5eb-8ed57e9def01", + "user_id" => 9, + "name" => %{"first" => "Ramona", "last" => "Floyd"}, + "age" => 22, + "location" => %{ + "state" => "Missouri", + "city" => "Foxworth", + "address" => %{"street" => "Lott Place", "number" => 1697}, + }, + "company" => "Manglo", + "email" => "ramonafloyd@manglo.com", + "manager" => true, + "twitter" => nil, + "favorites" => ["Lisp", "Erlang", "Python"], + "exists_array" => ["should", "exist", "array1"], + "complex_field_value" => "+-()%{}[]^~&&*||\"\\/? =>!", + "ordered" => true, + }, + %{ + "_id" => "e900001d-bc48-48a6-9b1a-ac9a1f5d1a03", + "user_id" => 10, + "name" => %{"first" => "Charmaine", "last" => "Mills"}, + "age" => 43, + "location" => %{ + "state" => "New Hampshire", + "city" => "Kiskimere", + "address" => %{"street" => "Nostrand Avenue", "number" => 4503}, + }, + "company" => "Lyria", + "email" => "charmainemills@lyria.com", + "manager" => true, + "favorites" => ["Erlang", "Erlang"], + "exists_array" => ["should", "exist", "array2"], + "ordered" => 9, + }, + %{ + "_id" => "b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4", + "user_id" => 11, + "name" => %{"first" => "Mathis", "last" => "Hernandez"}, + "age" => 75, + "location" => %{ + "state" => "Hawaii", + "city" => "Dupuyer", + "address" => %{"street" => "Bancroft Place", "number" => 2741}, + }, + "company" => "Affluex", + "email" => "mathishernandez@affluex.com", + "manager" => true, + "favorites" => ["Ruby", "Lisp", "C", "C++", "C++"], + "exists_object" => %{"should" => "object"}, + "ordered" => 10_000, + }, + %{ + "_id" => "5b61abc1-a3d3-4092-b9d7-ced90e675536", + "user_id" => 12, + "name" => %{"first" => "Patti", "last" => "Rosales"}, + "age" => 71, + "location" => %{ + "state" => "Pennsylvania", + "city" => "Juntura", + "address" => %{"street" => "Hunterfly Place", "number" => 7683}, + }, + "company" => "Oulu", + "email" => "pattirosales@oulu.com", + "manager" => true, + "favorites" => ["C", "Python", "Lisp"], + "exists_object" => %{"another" => "object"}, + "ordered" => "a", + }, + %{ + "_id" => "b1e70402-8add-4068-af8f-b4f3d0feb049", + "user_id" => 13, + "name" => %{"first" => "Whitley", "last" => "Harvey"}, + "age" => 78, + "location" => %{ + "state" => "Minnesota", + "city" => "Trail", + "address" => %{"street" => "Pleasant Place", "number" => 8766}, + }, + "company" => nil, + "email" => "whitleyharvey@fangold.com", + "manager" => false, + "twitter" => "@whitleyharvey", + "favorites" => ["C", "Ruby", "Ruby"], + "ordered" => "A", + }, + %{ + "_id" => "c78c529f-0b07-4947-90a6-d6b7ca81da62", + "user_id" => 14, + "name" => %{"first" => "Faith", "last" => "Hess"}, + "age" => 51, + "location" => %{ + "state" => "North Dakota", + "city" => "Axis", + "address" => %{"street" => "Brightwater Avenue", "number" => 1106}, + }, + "foo" => "bar car apple", + "company" => "Pharmex", + "email" => "faithhess@pharmex.com", + "favorites" => ["Erlang", "Python", "Lisp"], + "ordered" => "aa", + } + ] + + @users_docs [ + %{ + "_id" => "org.couchdb.user =>demo01", + "name" => "demo01", + "username" => "demo01", + "password" => "apple01", + "roles" => ["design"], + "order" => 1, + "type" => "user", + }, + %{ + "_id" => "org.couchdb.user =>demo02", + "name" => "demo02", + "username" => "demo02", + "password" => "apple02", + "roles" => ["reader"], + "order" => 2, + "type" => "user", + }, + %{ + "_id" => "org.couchdb.user =>demo03", + "name" => "demo03", + "username" => "demo03", + "password" => "apple03", + "roles" => ["reader", "writer"], + "order" => 3, + "type" => "user", + } + ] + + def setup_users(db) do + MangoDatabase.recreate(db) + MangoDatabase.save_docs(db, @users_docs) + end + + def setup(db, index_type \\ "view", partitioned \\ false) do + MangoDatabase.recreate(db, [partitioned: partitioned]) + docs = @docs + + if partitioned do + partition_docs = Enum.map(Enum.with_index(docs), fn {doc, index} -> + partition = rem(index, @partitions) + %{doc | "_id" => "#{partition}:#{doc["_id"]}"} + end) + MangoDatabase.save_docs(db, partition_docs) + else + MangoDatabase.save_docs(db, docs) + end + + case index_type do + "view" -> add_view_indexes(db) + "text" -> add_text_indexes(db) + "special" -> :ok + end + + :ok + end + + def len() do + length(@docs) + end + + defp add_view_indexes(db) do + indexes = [ + {["user_id"], "user_id"}, + {["name.last", "name.first"], "name"}, + {["age"], "age"}, + { + [ + "location.state", + "location.city", + "location.address.street", + "location.address.number", + ], + "location", + }, + {["company", "manager"], "company_and_manager"}, + {["manager"], "manager"}, + {["favorites"], "favorites"}, + {["favorites.3"], "favorites_3"}, + {["twitter"], "twitter"}, + {["ordered"], "ordered"}, + ] + + Enum.each(indexes, fn {idx, name} -> + MangoDatabase.create_index(db, idx, name: name, ddoc: name) + end) + end + + defp add_text_indexes(db) do + MangoDatabase.create_text_index(db) + end +end diff --git a/test/elixir/test/users_db_test.exs b/test/elixir/test/users_db_test.exs index bc09df9ba6c..2239eb4c247 100644 --- a/test/elixir/test/users_db_test.exs +++ b/test/elixir/test/users_db_test.exs @@ -126,7 +126,7 @@ defmodule UsersDbTest do headers: [authorization: "Basic Xzpf"] ) - assert resp.body["userCtx"]["name"] == :null + assert resp.body["userCtx"]["name"] == nil assert not Enum.member?(resp.body["info"], "authenticated") # ok, now create a conflicting edit on the jchris doc, and make sure there's no login. diff --git a/test/elixir/test/view_collation_raw_test.exs b/test/elixir/test/view_collation_raw_test.exs index ee272d72e8a..90f736621ba 100644 --- a/test/elixir/test/view_collation_raw_test.exs +++ b/test/elixir/test/view_collation_raw_test.exs @@ -13,7 +13,7 @@ defmodule ViewCollationRawTest do 3, 4, false, - :null, + nil, true, # Then objects, compared each key value in the list until different. @@ -109,7 +109,7 @@ defmodule ViewCollationRawTest do test "key query option", context do Enum.each(@values, fn value -> retry_until(fn -> - resp = Couch.get(url(context), query: %{:key => :jiffy.encode(value)}) + resp = Couch.get(url(context), query: %{:key => :jiffy.encode(value, [:use_nil])}) assert length(resp.body["rows"]) == 1 assert Enum.at(resp.body["rows"], 0)["key"] == convert(value) end) @@ -154,6 +154,6 @@ defmodule ViewCollationRawTest do end def convert(value) do - :jiffy.decode(:jiffy.encode(value), [:return_maps]) + :jiffy.decode(:jiffy.encode(value, [:use_nil]), [:return_maps, :use_nil]) end end diff --git a/test/elixir/test/view_collation_test.exs b/test/elixir/test/view_collation_test.exs index 7563ba41644..5f9a2ab97e6 100644 --- a/test/elixir/test/view_collation_test.exs +++ b/test/elixir/test/view_collation_test.exs @@ -8,7 +8,7 @@ defmodule ViewCollationTest do @values [ # Special values sort before all other types - :null, + nil, false, true, @@ -94,7 +94,7 @@ defmodule ViewCollationTest do test "key query option", context do Enum.each(@values, fn value -> retry_until(fn -> - resp = Couch.get(url(context), query: %{:key => :jiffy.encode(value)}) + resp = Couch.get(url(context), query: %{:key => :jiffy.encode(value, [:use_nil])}) assert length(resp.body["rows"]) == 1 assert Enum.at(resp.body["rows"], 0)["key"] == convert(value) end) @@ -139,6 +139,6 @@ defmodule ViewCollationTest do end def convert(value) do - :jiffy.decode(:jiffy.encode(value), [:return_maps]) + :jiffy.decode(:jiffy.encode(value, [:use_nil]), [:return_maps, :use_nil]) end end diff --git a/test/elixir/test/view_errors_test.exs b/test/elixir/test/view_errors_test.exs index d45eb7c3f7f..bf563127774 100644 --- a/test/elixir/test/view_errors_test.exs +++ b/test/elixir/test/view_errors_test.exs @@ -20,7 +20,7 @@ defmodule ViewErrorsTest do # being included in the view results as null results = query(db_name, map_fun) assert results["total_rows"] == 1 - assert Enum.at(results["rows"], 0)["key"] == :null + assert Enum.at(results["rows"], 0)["key"] == nil end @tag :with_db @@ -62,7 +62,7 @@ defmodule ViewErrorsTest do |> Map.get("key") |> Enum.at(1) - assert key == :null + assert key == nil end @tag :with_db diff --git a/test/elixir/test/view_multi_key_all_docs_test.exs b/test/elixir/test/view_multi_key_all_docs_test.exs index d9fa41e2375..4ca64363f18 100644 --- a/test/elixir/test/view_multi_key_all_docs_test.exs +++ b/test/elixir/test/view_multi_key_all_docs_test.exs @@ -27,7 +27,7 @@ defmodule ViewMultiKeyAllDocsTest do test "keys in GET parameters", context do db_name = context[:db_name] - resp = all_docs(db_name, keys: :jiffy.encode(@keys)) + resp = all_docs(db_name, keys: :jiffy.encode(@keys, [:use_nil])) assert resp.status_code == 200 rows = resp.body["rows"] assert length(rows) == length(@keys) @@ -47,7 +47,7 @@ defmodule ViewMultiKeyAllDocsTest do test "keys in GET parameters (limit)", context do db_name = context[:db_name] - resp = all_docs(db_name, limit: 1, keys: :jiffy.encode(@keys)) + resp = all_docs(db_name, limit: 1, keys: :jiffy.encode(@keys, [:use_nil])) assert resp.status_code == 200 rows = resp.body["rows"] assert length(rows) == 1 @@ -68,7 +68,7 @@ defmodule ViewMultiKeyAllDocsTest do test "keys in GET parameters (skip)", context do db_name = context[:db_name] - resp = all_docs(db_name, skip: 2, keys: :jiffy.encode(@keys)) + resp = all_docs(db_name, skip: 2, keys: :jiffy.encode(@keys, [:use_nil])) assert resp.status_code == 200 rows = resp.body["rows"] assert length(rows) == 3 @@ -90,7 +90,7 @@ defmodule ViewMultiKeyAllDocsTest do test "keys in GET parameters (descending)", context do db_name = context[:db_name] - resp = all_docs(db_name, descending: true, keys: :jiffy.encode(@keys)) + resp = all_docs(db_name, descending: true, keys: :jiffy.encode(@keys, [:use_nil])) assert resp.status_code == 200 rows = resp.body["rows"] assert length(rows) == length(@keys) @@ -119,7 +119,7 @@ defmodule ViewMultiKeyAllDocsTest do db_name = context[:db_name] resp = - all_docs(db_name, descending: "true", skip: 3, limit: 1, keys: :jiffy.encode(@keys)) + all_docs(db_name, descending: "true", skip: 3, limit: 1, keys: :jiffy.encode(@keys, [:use_nil])) assert resp.status_code == 200 rows = resp.body["rows"] diff --git a/test/elixir/test/view_multi_key_design_test.exs b/test/elixir/test/view_multi_key_design_test.exs index c33491620e1..0af99b4072e 100644 --- a/test/elixir/test/view_multi_key_design_test.exs +++ b/test/elixir/test/view_multi_key_design_test.exs @@ -53,7 +53,7 @@ defmodule ViewMultiKeyDesignTest do test "keys in GET parameters", context do db_name = context[:db_name] - resp = view(db_name, "test/all_docs", keys: :jiffy.encode(@keys)) + resp = view(db_name, "test/all_docs", keys: :jiffy.encode(@keys, [:use_nil])) rows = resp.body["rows"] assert length(rows) == length(@keys) assert Enum.all?(rows, &Enum.member?(@keys, &1["key"])) @@ -80,7 +80,7 @@ defmodule ViewMultiKeyDesignTest do test "keys in GET body (group)", context do db_name = context[:db_name] - resp = view(db_name, "test/summate", group: true, keys: :jiffy.encode(@keys)) + resp = view(db_name, "test/summate", group: true, keys: :jiffy.encode(@keys, [:use_nil])) rows = resp.body["rows"] assert length(rows) == length(@keys) assert Enum.all?(rows, &Enum.member?(@keys, &1["key"])) @@ -117,10 +117,10 @@ defmodule ViewMultiKeyDesignTest do db_name = context[:db_name] badargs = [ - [startkey: 0, keys: :jiffy.encode(@keys)], - [endkey: 0, keys: :jiffy.encode(@keys)], - [key: 0, keys: :jiffy.encode(@keys)], - [group_level: 2, keys: :jiffy.encode(@keys)] + [startkey: 0, keys: :jiffy.encode(@keys, [:use_nil])], + [endkey: 0, keys: :jiffy.encode(@keys, [:use_nil])], + [key: 0, keys: :jiffy.encode(@keys, [:use_nil])], + [group_level: 2, keys: :jiffy.encode(@keys, [:use_nil])] ] Enum.each(badargs, fn args -> @@ -135,7 +135,7 @@ defmodule ViewMultiKeyDesignTest do resp = Couch.get("/#{db_name}/_design/test/_view/summate", - query: [keys: :jiffy.encode(@keys)], + query: [keys: :jiffy.encode(@keys, [:use_nil])], body: %{"keys" => @keys} ) @@ -148,7 +148,7 @@ defmodule ViewMultiKeyDesignTest do resp = view(db_name, "test/summate", [reduce: false], @keys) assert length(resp.body["rows"]) == 5 - resp = view(db_name, "test/summate", reduce: false, keys: :jiffy.encode(@keys)) + resp = view(db_name, "test/summate", reduce: false, keys: :jiffy.encode(@keys, [:use_nil])) assert length(resp.body["rows"]) == 5 end @@ -191,7 +191,7 @@ defmodule ViewMultiKeyDesignTest do assert length(rows) == 1 assert Enum.at(rows, 0)["key"] == 10 - resp = view(db_name, "test/all_docs", limit: 1, keys: :jiffy.encode(@keys)) + resp = view(db_name, "test/all_docs", limit: 1, keys: :jiffy.encode(@keys, [:use_nil])) rows = resp.body["rows"] assert length(rows) == 1 assert Enum.at(rows, 0)["key"] == 10 diff --git a/test/elixir/test/view_offsets_test.exs b/test/elixir/test/view_offsets_test.exs index edb5a58f66c..34f2aba71db 100644 --- a/test/elixir/test/view_offsets_test.exs +++ b/test/elixir/test/view_offsets_test.exs @@ -51,7 +51,7 @@ defmodule ViewOffsetTest do |> Enum.each(fn [start_key, offset] -> result = view(db_name, "test/offset", %{ - "startkey" => :jiffy.encode(start_key), + "startkey" => :jiffy.encode(start_key, [:use_nil]), "descending" => true })