diff --git a/cpanfile b/cpanfile index 53aa6768d..3e0296b6a 100644 --- a/cpanfile +++ b/cpanfile @@ -122,6 +122,7 @@ requires 'Scalar::Util', '1.62'; # Moose requires 'Search::Elasticsearch' => '8.12'; requires 'Search::Elasticsearch::Client::2_0' => '6.81'; requires 'Search::Elasticsearch::Client::5_0' => '6.81'; +requires 'Search::Elasticsearch::Client::7_0' => '8.12'; requires 'Throwable::Error'; requires 'Term::Size::Any'; # for Catalyst requires 'Text::CSV_XS'; diff --git a/es/account/mapping.json b/es/account/mapping.json index 0b4e92567..1d89039a6 100644 --- a/es/account/mapping.json +++ b/es/account/mapping.json @@ -5,39 +5,27 @@ "dynamic": true, "properties": { "client": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "token": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, "code": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "identity": { "dynamic": false, "properties": { "key": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/account/settings.json b/es/account/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/account/settings.json +++ b/es/account/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/author/mapping.json b/es/author/mapping.json index ad9278a7f..d643dfe50 100644 --- a/es/author/mapping.json +++ b/es/author/mapping.json @@ -5,66 +5,46 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "blog": { "dynamic": true, "properties": { "feed": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, "city": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "country": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "donation": { "dynamic": true, "properties": { "id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, "email": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "gravatar_url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "is_pause_custodial_account": { "type": "boolean" @@ -76,34 +56,24 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "pauseid": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "perlmongers": { "dynamic": true, "properties": { "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, @@ -115,43 +85,31 @@ "fields": { "analyzed": { "analyzer": "simple", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } }, "type": "nested" }, "region": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "updated": { "format": "strict_date_optional_time||epoch_millis", "type": "date" }, "user": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "website": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/author/settings.json b/es/author/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/author/settings.json +++ b/es/author/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/contributor/mapping.json b/es/contributor/mapping.json index 0405a4494..72017a284 100644 --- a/es/contributor/mapping.json +++ b/es/contributor/mapping.json @@ -2,34 +2,22 @@ "dynamic": false, "properties": { "distribution": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, - "pauseid": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "email": { + "type": "keyword" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, - "email": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "pauseid": { + "type": "keyword" }, "release_author": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "release_name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/contributor/settings.json b/es/contributor/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/contributor/settings.json +++ b/es/contributor/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/cover/mapping.json b/es/cover/mapping.json index e0a6a6114..274adddc1 100644 --- a/es/cover/mapping.json +++ b/es/cover/mapping.json @@ -22,19 +22,13 @@ } }, "distribution": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "release": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/cover/settings.json b/es/cover/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/cover/settings.json +++ b/es/cover/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/cve/mapping.json b/es/cve/mapping.json index 4f3492037..8ec674c24 100644 --- a/es/cve/mapping.json +++ b/es/cve/mapping.json @@ -2,40 +2,35 @@ "dynamic": false, "properties": { "affected_versions": { - "type": "string" + "type": "text" }, "cpansa_id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "cves": { - "type": "string" + "type": "text" }, "description": { - "type": "string" + "type": "text" }, "distribution": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "references": { - "type": "string" + "type": "text" }, "releases": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "reported": { "format": "strict_date_optional_time||epoch_millis", "type": "date" }, "severity": { - "type": "string" + "type": "text" }, "versions": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/cve/settings.json b/es/cve/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/cve/settings.json +++ b/es/cve/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/distribution/mapping.json b/es/distribution/mapping.json index 333927a24..1e652f69b 100644 --- a/es/distribution/mapping.json +++ b/es/distribution/mapping.json @@ -17,9 +17,7 @@ "type": "integer" }, "source": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, @@ -48,9 +46,7 @@ "type": "integer" }, "source": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "stalled": { "type": "integer" @@ -63,26 +59,18 @@ "dynamic": true, "properties": { "cygwin": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "debian": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "fedora": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "repo": { "dynamic": true, diff --git a/es/distribution/settings.json b/es/distribution/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/distribution/settings.json +++ b/es/distribution/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/favorite/mapping.json b/es/favorite/mapping.json index aeff5b2cb..0aa0f6e88 100644 --- a/es/favorite/mapping.json +++ b/es/favorite/mapping.json @@ -2,33 +2,23 @@ "dynamic": false, "properties": { "author": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "date": { "format": "strict_date_optional_time||epoch_millis", "type": "date" }, "distribution": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "release": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "user": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/favorite/settings.json b/es/favorite/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/favorite/settings.json +++ b/es/favorite/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/file/mapping.json b/es/file/mapping.json index 5374d9f41..12dc23970 100644 --- a/es/file/mapping.json +++ b/es/file/mapping.json @@ -5,21 +5,15 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "author": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "authorized": { "type": "boolean" @@ -35,14 +29,10 @@ "type": "boolean" }, "description": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "text" }, "dir": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "directory": { "type": "boolean" @@ -54,74 +44,62 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "documentation": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "edge": { "analyzer": "edge", "store": true, - "type": "string" + "type": "text" }, "edge_camelcase": { "analyzer": "edge_camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "documentation_length": { "type": "integer" }, "download_url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "indexed": { "type": "boolean" @@ -130,21 +108,17 @@ "type": "integer" }, "maturity": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "mime": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "module": { "dynamic": false, "include_in_root": true, "properties": { "associated_pod": { - "type": "string" + "type": "text" }, "authorized": { "type": "boolean" @@ -156,31 +130,25 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version_numified": { "type": "float" @@ -189,59 +157,44 @@ "type": "nested" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "path": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "pod": { "analyzer": "standard", - "doc_values": false, "fields": { "analyzed": { "analyzer": "standard", - "doc_values": false, - "index": "analyzed", - "type": "string" + "type": "text" } }, - "index": "analyzed", - "type": "string" + "type": "text" }, "pod_lines": { - "doc_values": true, - "ignore_above": 2048, - "index": "no", - "type": "string" + "type": "keyword" }, "release": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "sloc": { "type": "integer" @@ -270,9 +223,7 @@ } }, "status": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "suggest": { "analyzer": "simple", @@ -282,9 +233,7 @@ "type": "completion" }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version_numified": { "type": "float" diff --git a/es/file/settings.json b/es/file/settings.json index 1d4845446..1cdf6e2f0 100644 --- a/es/file/settings.json +++ b/es/file/settings.json @@ -47,9 +47,6 @@ } } }, - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/mirror/mapping.json b/es/mirror/mapping.json index 09f2d85e1..eaebd2742 100644 --- a/es/mirror/mapping.json +++ b/es/mirror/mapping.json @@ -2,47 +2,33 @@ "dynamic": false, "properties": { "A_or_CNAME": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "aka_name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "ccode": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "city": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "contact": { "dynamic": false, "properties": { "contact_site": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "contact_user": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } }, @@ -50,51 +36,35 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "country": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "dnsrr": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "freq": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "ftp": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "http": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "inceptdate": { "format": "strict_date_optional_time||epoch_millis", @@ -104,63 +74,45 @@ "type": "geo_point" }, "name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "note": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "org": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "region": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "reitredate": { "format": "strict_date_optional_time||epoch_millis", "type": "date" }, "rsync": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "src": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "tz": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/mirror/settings.json b/es/mirror/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/mirror/settings.json +++ b/es/mirror/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/package/mapping.json b/es/package/mapping.json index 82dacee1e..0248dfe82 100644 --- a/es/package/mapping.json +++ b/es/package/mapping.json @@ -2,34 +2,22 @@ "dynamic": false, "properties": { "author": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "dist_version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "distribution": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "file": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "module_name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/package/settings.json b/es/package/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/package/settings.json +++ b/es/package/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/permission/mapping.json b/es/permission/mapping.json index 688f56bcd..31c2e36b6 100644 --- a/es/permission/mapping.json +++ b/es/permission/mapping.json @@ -2,19 +2,13 @@ "dynamic": false, "properties": { "co_maintainers": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "module_name": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "owner": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } diff --git a/es/permission/settings.json b/es/permission/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/permission/settings.json +++ b/es/permission/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/release/mapping.json b/es/release/mapping.json index fb75ac875..4ab1b1a28 100644 --- a/es/release/mapping.json +++ b/es/release/mapping.json @@ -5,44 +5,30 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "archive": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "author": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "authorized": { "type": "boolean" }, "changes_file": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "checksum_md5": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "checksum_sha256": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "date": { "format": "strict_date_optional_time||epoch_millis", @@ -53,24 +39,16 @@ "include_in_root": true, "properties": { "module": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "phase": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "relationship": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } }, "type": "nested" @@ -82,134 +60,100 @@ "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "download_url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "first": { "type": "boolean" }, "id": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "license": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "main_module": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "maturity": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "name": { "fields": { "analyzed": { "analyzer": "standard", - "fielddata": { - "format": "disabled" - }, + "fielddata": false, "store": true, - "type": "string" + "type": "text" }, "camelcase": { "analyzer": "camelcase", "store": true, - "type": "string" + "type": "text" }, "lowercase": { "analyzer": "lowercase", "store": true, - "type": "string" + "type": "text" } }, - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "provides": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "resources": { "dynamic": true, "include_in_root": true, "properties": { "bugtracker": { - "dynamic": "true", + "dynamic": true, "include_in_root": true, "properties": { "mailto": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "web": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } }, "type": "nested" }, "homepage": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "license": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "repository": { - "dynamic": "true", + "dynamic": true, "include_in_root": true, "properties": { "type": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "url": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "web": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" } }, "type": "nested" @@ -238,9 +182,7 @@ } }, "status": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "tests": { "dynamic": true, @@ -260,9 +202,7 @@ } }, "version": { - "ignore_above": 2048, - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "version_numified": { "type": "float" diff --git a/es/release/settings.json b/es/release/settings.json index 1d4845446..1cdf6e2f0 100644 --- a/es/release/settings.json +++ b/es/release/settings.json @@ -47,9 +47,6 @@ } } }, - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/session/settings.json b/es/session/settings.json index c90ef6cba..bbb95c38b 100644 --- a/es/session/settings.json +++ b/es/session/settings.json @@ -1,7 +1,4 @@ { - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/es/settings.json b/es/settings.json index 1d4845446..1cdf6e2f0 100644 --- a/es/settings.json +++ b/es/settings.json @@ -47,9 +47,6 @@ } } }, - "mapper": { - "dynamic": "false" - }, "number_of_replicas": 1, "number_of_shards": 1, "refresh_interval": "1s" diff --git a/lib/ElasticSearchX/Model/Document/Role.pm b/lib/ElasticSearchX/Model/Document/Role.pm new file mode 100644 index 000000000..6d2b58be2 --- /dev/null +++ b/lib/ElasticSearchX/Model/Document/Role.pm @@ -0,0 +1,24 @@ +package ElasticSearchX::Model::Document::Role; +use strict; +use warnings; + +use MetaCPAN::Model::Hacks; + +no warnings 'redefine'; + +my $_put = \&_put; +*_put = sub { + my ($self) = @_; + my $es = $self->index->model->es; + + my %return = &$_put; + + if ( $es->api_version le '6_0' ) { + return %return; + } + + delete $return{type}; + return %return; +}; + +1; diff --git a/lib/MetaCPAN/ESConfig.pm b/lib/MetaCPAN/ESConfig.pm index d4ae3feb8..e84f1d928 100644 --- a/lib/MetaCPAN/ESConfig.pm +++ b/lib/MetaCPAN/ESConfig.pm @@ -11,7 +11,7 @@ use Exporter qw( import ); use Hash::Merge::Simple qw( merge ); use MetaCPAN::Server::Config (); use MetaCPAN::Types::TypeTiny qw( Defined HashRef ); -use MetaCPAN::Util qw( root_dir ); +use MetaCPAN::Util qw( root_dir true false ); use Module::Runtime qw( $module_name_rx require_module ); const my %config => merge( @@ -187,19 +187,57 @@ sub _load_es_data ( $location, $def_sub ) { return Cpanel::JSON::XS::decode_json($data); } -sub mapping ( $self, $doc ) { +sub _walk : prototype(&$); + +sub _walk : prototype(&$) { + my ( $cb, $data ) = @_; + if ( ref $data eq 'HASH' ) { + $cb->($data); + _walk( \&$cb, $data->{$_} ) for keys %$data; + } + elsif ( ref $data eq 'ARRAY' ) { + $cb->($data); + _walk( \&$cb, $_ ) for @$data; + } +} + +sub mapping ( $self, $doc, $version ) { my $doc_data = $self->documents->{$doc} or croak "unknown document $doc"; - return _load_es_data( $doc_data->{mapping}, 'mapping' ); + my $data = _load_es_data( $doc_data->{mapping}, 'mapping' ); + if ( $version && $version eq '2_0' ) { + _walk( + sub { + my ($d) = @_; + if ( my $type = $d->{type} ) { + if ( $type eq 'keyword' ) { + $d->{type} = 'string'; + $d->{index} = 'not_analyzed'; + $d->{ignore_above} = 2048; + } + elsif ( $type eq 'text' ) { + $d->{type} = 'string'; + if ( exists $d->{fielddata} && !$d->{fielddata} ) { + $d->{fielddata} = { format => false }; + } + } + } + + }, + $data + ); + } + return $data; } -sub index_settings ( $self, $doc ) { +sub index_settings ( $self, $doc, $version = undef ) { my $documents = $self->documents; my $doc_data = exists $documents->{$doc} && $documents->{$doc} or return {}; my $settings = exists $doc_data->{settings} && $doc_data->{settings} or return {}; - return _load_es_data( $settings, 'settings' ); + my $data = _load_es_data( $settings, 'settings' ); + return $data; } sub doc_path ( $self, $doc ) { diff --git a/lib/MetaCPAN/Model.pm b/lib/MetaCPAN/Model.pm index fd17cab71..f45cdea28 100644 --- a/lib/MetaCPAN/Model.pm +++ b/lib/MetaCPAN/Model.pm @@ -28,7 +28,8 @@ for my $index ( sort keys %indexes ) { sub doc { my ( $self, $doc ) = @_; my $doc_config = es_config->documents->{$doc}; - return $self->index( $doc_config->{index} )->type( $doc_config->{type} ); + return $self->index( $doc_config->{index} ) + ->type( $doc_config->{type} // $doc_config->{index} ); } __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/Model/ESWrapper.pm b/lib/MetaCPAN/Model/ESWrapper.pm new file mode 100644 index 000000000..9cc17b497 --- /dev/null +++ b/lib/MetaCPAN/Model/ESWrapper.pm @@ -0,0 +1,61 @@ +package MetaCPAN::Model::ESWrapper; +use strict; +use warnings; + +use MetaCPAN::Types::TypeTiny qw(ES); + +sub new { + my ( $class, $es ) = @_; + if ( $es->api_version le '6_0' ) { + return $es; + } + return bless { es => ES->assert_coerce($es) }, $class; +} + +sub DESTROY { } + +sub AUTOLOAD { + my $sub = our $AUTOLOAD =~ s/.*:://r; + my $self = shift; + $self->{es}->$sub(@_); +} + +sub _args { + my $self = shift; + if ( @_ == 1 ) { + return ( $self, %{ $_[0] } ); + } + return ( $self, @_ ); +} + +sub count { + my ( $self, %args ) = &_args; + delete $args{type}; + $self->{es}->count(%args); +} + +sub get { + my ( $self, %args ) = &_args; + delete $args{type}; + $self->{es}->get(%args); +} + +sub delete { + my ( $self, %args ) = &_args; + delete $args{type}; + $self->{es}->delete(%args); +} + +sub search { + my ( $self, %args ) = &_args; + delete $args{type}; + $self->{es}->search(%args); +} + +sub scroll_helper { + my ( $self, %args ) = &_args; + delete $args{type}; + $self->{es}->scroll_helper(%args); +} + +1; diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index 9b550d43c..b45011e25 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -898,7 +898,7 @@ sub find_download_url { ], ( exists $version_filters->{must_not} - ? ( must_not => [ $version_filters->{must_not} ] ) + ? ( must_not => [ @{ $version_filters->{must_not} } ] ) : () ) } diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm index 90824d22c..3ed00e450 100644 --- a/lib/MetaCPAN/Role/Script.pm +++ b/lib/MetaCPAN/Role/Script.pm @@ -2,14 +2,15 @@ package MetaCPAN::Role::Script; use Moose::Role; -use Carp (); -use IO::Prompt::Tiny qw( prompt ); -use Log::Contextual qw( :log :dlog ); -use MetaCPAN::Model (); -use MetaCPAN::Types::TypeTiny qw( AbsPath Bool ES HashRef Int Path Str ); -use MetaCPAN::Util qw( root_dir ); -use Mojo::Server (); -use Term::ANSIColor qw( colored ); +use Carp (); +use IO::Prompt::Tiny qw( prompt ); +use Log::Contextual qw( :log :dlog ); +use MetaCPAN::Model (); +use MetaCPAN::Types::TypeTiny qw( AbsPath Bool ES HashRef Int Path Str ); +use MetaCPAN::Util qw( root_dir ); +use Mojo::Server (); +use Term::ANSIColor qw( colored ); +use MetaCPAN::Model::ESWrapper (); use MooseX::Getopt::OptionTypeMap (); for my $type ( Path, AbsPath, ES ) { @@ -136,7 +137,9 @@ sub _build_model { my $self = shift; # es provided by ElasticSearchX::Model::Role - return MetaCPAN::Model->new( es => $self->es ); + + my $es = MetaCPAN::Model::ESWrapper->new( $self->es ); + return MetaCPAN::Model->new( es => $es ); } sub _build_ua { diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index 04d9225a5..801b05a39 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -41,14 +41,6 @@ has arg_verify_mapping => ( documentation => 'verify deployed index structure against definition', ); -has arg_list_types => ( - init_arg => 'list_types', - is => 'ro', - isa => Bool, - default => 0, - documentation => 'list available document type names', -); - has arg_cluster_info => ( init_arg => 'show_cluster_info', is => 'ro', @@ -57,80 +49,6 @@ has arg_cluster_info => ( documentation => 'show basic info about cluster and indices', ); -has arg_create_index => ( - init_arg => 'create_index', - is => 'ro', - isa => Str, - default => "", - documentation => 'create a new empty index (copy mappings)', -); - -has arg_update_index => ( - init_arg => 'update_index', - is => 'ro', - isa => Str, - default => "", - documentation => 'update existing index (add mappings)', -); - -has patch_mapping => ( - is => 'ro', - isa => Str, - default => "{}", - documentation => 'type mapping patches', -); - -has skip_existing_mapping => ( - is => 'ro', - isa => Bool, - default => 0, - documentation => 'do NOT copy mappings other than patch_mapping', -); - -has copy_from_index => ( - is => 'ro', - isa => Str, - documentation => 'index to copy type from', -); - -has copy_to_index => ( - is => 'ro', - isa => Str, - default => "", - documentation => 'index to copy type to', -); - -has arg_copy_type => ( - init_arg => 'copy_type', - is => 'ro', - isa => Str, - default => "", - documentation => 'type to copy', -); - -has copy_query => ( - is => 'ro', - isa => Str, - default => "", - documentation => 'match query (default: monthly time slices, ' - . ' if provided must be a valid json query OR "match_all")', -); - -has reindex => ( - is => 'ro', - isa => Bool, - default => 0, - documentation => 'reindex data from source index for exact mapping types', -); - -has arg_delete_index => ( - init_arg => 'delete_index', - is => 'ro', - isa => Str, - default => "", - documentation => 'delete an existing index', -); - has arg_await_timeout => ( init_arg => 'await', is => 'ro', @@ -140,13 +58,6 @@ has arg_await_timeout => ( 'seconds before connection is considered failed with timeout', ); -has delete_from_type => ( - is => 'ro', - isa => Str, - default => "", - documentation => 'delete data from an existing type', -); - has cluster_info => ( isa => HashRef, traits => ['Hash'], @@ -168,22 +79,7 @@ sub run { # Wait for the ElasticSearch Engine to become ready if ( $self->await ) { - if ( $self->arg_delete_index ) { - $self->delete_index; - } - elsif ( $self->arg_create_index ) { - $self->create_index; - } - elsif ( $self->arg_update_index ) { - $self->update_index; - } - elsif ( $self->copy_to_index ) { - $self->copy_type; - } - elsif ( $self->delete_from_type ) { - $self->empty_type; - } - elsif ( $self->arg_deploy_mapping ) { + if ( $self->arg_deploy_mapping ) { if ( $self->arg_delete_all ) { $self->check_health; $self->delete_all; @@ -202,10 +98,6 @@ sub run { } } - if ( $self->arg_list_types ) { - $self->list_types; - } - if ( $self->arg_cluster_info ) { $self->check_health; $self->show_info; @@ -216,37 +108,6 @@ sub run { return ( $self->exit_code == 0 ? 1 : 0 ); } -sub _check_index_exists { - my ( $self, $name, $expected ) = @_; - my $exists = $self->es->indices->exists( index => $name ); - - if ( $exists and !$expected ) { - log_error {"Index already exists: $name"}; - - #Set System Error: 1 - EPERM - Operation not permitted - $self->exit_code(1); - $self->handle_error( "Conflicting index: $name", 1 ); - } - - if ( !$exists and $expected ) { - log_error {"Index doesn't exists: $name"}; - - #Set System Error: 1 - EPERM - Operation not permitted - $self->exit_code(1); - $self->handle_error( "Missing index: $name", 1 ); - } -} - -sub delete_index { - my $self = shift; - my $name = $self->arg_delete_index; - - $self->_check_index_exists( $name, EXPECTED ); - $self->are_you_sure("Index $name will be deleted !!!"); - - $self->_delete_index($name); -} - sub delete_all { my $self = $_[0]; my $runtime_environment = 'production'; @@ -295,211 +156,6 @@ sub _delete_index { return $exists; } -sub update_index { - my $self = shift; - my $name = $self->arg_update_index; - - $self->_check_index_exists( $name, EXPECTED ); - $self->are_you_sure("Index $name will be updated !!!"); - - die "update_index requires patch_mapping\n" - unless $self->patch_mapping; - - my $patch_mapping = decode_json $self->patch_mapping; - my @patch_types = sort keys %{$patch_mapping}; - my $mapping = +{ map { $_ => $patch_mapping->{$_} } @patch_types }; - - log_info {"Updating mapping for index: $name"}; - - for my $type ( sort keys %{$mapping} ) { - log_info {"Adding mapping to index: $type"}; - $self->es->indices->put_mapping( - index => $name, - type => $type, - body => { $type => $mapping->{$type} }, - ); - } - - log_info {"Done."}; -} - -sub create_index { - my $self = shift; - - my $dst_idx = $self->arg_create_index; - $self->_check_index_exists( $dst_idx, NOT_EXPECTED ); - - my $patch_mapping = decode_json $self->patch_mapping; - my @patch_types = sort keys %{$patch_mapping}; - my $index_settings = es_config->index_settings($dst_idx); - my $mapping = +{}; - - # create the new index with the copied settings - log_info {"Creating index: $dst_idx"}; - $self->es->indices->create( - index => $dst_idx, - body => { - settings => $index_settings, - }, - ); - - # override with new type mapping - if ( $self->patch_mapping ) { - for my $type (@patch_types) { - log_info {"Patching mapping for type: $type"}; - $mapping->{$type} = $patch_mapping->{$type}; - } - } - - # add the mappings to the index - for my $type ( sort keys %{$mapping} ) { - log_info {"Adding mapping to index: $type"}; - $self->es->indices->put_mapping( - index => $dst_idx, - type => $type, - body => { $type => $mapping->{$type} }, - ); - } - - # copy the data to the non-altered types - if ( $self->reindex ) { - for my $type ( - grep { !exists $patch_mapping->{$_} } - sort keys %{$mapping} - ) - { - log_info {"Re-indexing data to index $dst_idx from type: $type"}; - $self->copy_type( $dst_idx, $type ); - } - } - - log_info { - "Done. you can now fill the data for the altered types: (" - . join( ',', @patch_types ) . ")" - } - if @patch_types; -} - -sub copy_type { - my ( $self, $index, $type ) = @_; - my $from_index = $self->copy_from_index - or die "can't copy without a source index"; - $index //= $self->copy_to_index - or die "can't copy without a destination index"; - - $self->_check_index_exists( $from_index, EXPECTED ); - $self->_check_index_exists( $index, EXPECTED ); - $type //= $self->arg_copy_type; - $type or die "can't copy without a type\n"; - - my $arg_query = $self->copy_query; - my $query - = $arg_query eq 'match_all' - ? +{ match_all => {} } - : undef; - - if ( $arg_query and !$query ) { - eval { - $query = decode_json $arg_query; - 1; - } or do { - my $err = $@ || 'zombie error'; - die $err; - }; - } - - return $self->_copy_slice( $query, $from_index, $index, $type ) if $query; - - # else ... do copy by monthly slices - - my $dt = DateTime->new( year => 1994, month => 1 ); - my $end_time = DateTime->now()->add( months => 1 ); - - while ( $dt < $end_time ) { - my $gte = $dt->strftime("%Y-%m"); - $dt->add( months => 1 ); - my $lt = $dt->strftime("%Y-%m"); - - my $q = +{ range => { date => { gte => $gte, lt => $lt } } }; - - log_info {"copying data for month: $gte"}; - eval { - $self->_copy_slice( $q, $from_index, $index, $type ); - 1; - } or do { - my $err = $@ || 'zombie error'; - warn $err; - }; - } -} - -sub _copy_slice { - my ( $self, $query, $from_index, $index, $type ) = @_; - - my $scroll = $self->es->scroll_helper( - size => 250, - scroll => '10m', - index => $from_index, - type => $type, - body => { - query => $query, - sort => '_doc', - }, - ); - - my $bulk = $self->es->bulk_helper( - index => $index, - type => $type, - max_count => 500, - ); - - while ( my $search = $scroll->next ) { - $bulk->create( { - id => $search->{_id}, - source => $search->{_source} - } ); - } - - $bulk->flush; -} - -sub empty_type { - my $self = shift; - my $type = $self->delete_from_type; - log_info {"Emptying type: $type"}; - - my $bulk - = $self->es->bulk_helper( es_doc_path($type), max_count => 500, ); - - my $scroll = $self->es->scroll_helper( - size => 250, - scroll => '10m', - es_doc_path($type), - body => { - query => { match_all => {} }, - sort => '_doc', - }, - ); - - my @ids; - while ( my $search = $scroll->next ) { - push @ids => $search->{_id}; - log_debug { "deleting id=" . $search->{_id} }; - if ( @ids == 500 ) { - $bulk->delete_ids(@ids); - @ids = (); - } - } - $bulk->delete_ids(@ids); - - $bulk->flush; -} - -sub list_types { - my $self = shift; - print "$_\n" for sort keys %{ es_config->documents }; -} - sub show_info { my $self = $_[0]; my $info_rs = { @@ -510,24 +166,26 @@ sub show_info { } sub _build_index_config { - my $self = $_[0]; - my $docs = es_config->documents; - my $indices = {}; + my $self = $_[0]; + my $docs = es_config->documents; + my $indices = {}; + my $api_version = $self->es->api_version; for my $name ( sort keys %$docs ) { my $doc = $docs->{$name}; my $index = $doc->{index} or die "no index defined for $name documents"; - my $type = $doc->{type} - or die "no type defined for $name documents"; die "$index specified for multiple documents" if $indices->{$index}; - my $mapping = es_config->mapping($name); - my $settings = es_config->index_settings($name); + my $mapping = es_config->mapping( $name, $api_version ); + my $settings = es_config->index_settings( $name, $api_version ); + if ( $api_version le '6_0' ) { + my $type = $doc->{type} + or die "no type defined for $name documents"; + $mapping = { $type => $mapping }; + } $indices->{$index} = { settings => $settings, - mappings => { - $type => $mapping, - }, + mappings => $mapping, }; } @@ -945,15 +603,6 @@ MetaCPAN::Script::Mapping - Script to set the index and mapping the types # bin/metacpan mapping --delete # bin/metacpan mapping --delete --all # deletes ALL indices in the cluster # bin/metacpan mapping --verify # compare deployed indices with project definitions - # bin/metacpan mapping --list_types - # bin/metacpan mapping --delete_index xxx - # bin/metacpan mapping --create_index xxx --reindex - # bin/metacpan mapping --create_index xxx --reindex --patch_mapping '{"distribution":{"dynamic":"false","properties":{"name":{"index":"not_analyzed","ignore_above":2048,"type":"string"},"river":{"properties":{"total":{"type":"integer"},"immediate":{"type":"integer"},"bucket":{"type":"integer"}},"dynamic":"true"},"bugs":{"properties":{"rt":{"dynamic":"true","properties":{"rejected":{"type":"integer"},"closed":{"type":"integer"},"open":{"type":"integer"},"active":{"type":"integer"},"patched":{"type":"integer"},"source":{"type":"string","ignore_above":2048,"index":"not_analyzed"},"resolved":{"type":"integer"},"stalled":{"type":"integer"},"new":{"type":"integer"}}},"github":{"dynamic":"true","properties":{"active":{"type":"integer"},"open":{"type":"integer"},"closed":{"type":"integer"},"source":{"type":"string","index":"not_analyzed","ignore_above":2048}}}},"dynamic":"true"}}}}' - # bin/metacpan mapping --create_index xxx --patch_mapping '{...mapping...}' --skip_existing_mapping - # bin/metacpan mapping --update_index xxx --patch_mapping '{...mapping...}' - # bin/metacpan mapping --copy_to_index xxx --copy_type release - # bin/metacpan mapping --copy_to_index xxx --copy_type release --copy_query '{"range":{"date":{"gte":"2016-01","lt":"2017-01"}}}' - # bin/metacpan mapping --delete_from_type xxx # empty the type =head1 DESCRIPTION diff --git a/lib/MetaCPAN/Script/Role/Contributor.pm b/lib/MetaCPAN/Script/Role/Contributor.pm index 5aa0b928e..c2384f488 100644 --- a/lib/MetaCPAN/Script/Role/Contributor.pm +++ b/lib/MetaCPAN/Script/Role/Contributor.pm @@ -96,7 +96,7 @@ sub release_contributor_update_actions { qw(pauseid name email) }; } @$contribs; - push @actions, map +{ create => { _source => $_ } }, @docs; + push @actions, map +{ index => { source => $_ } }, @docs; return \@actions; } diff --git a/lib/MetaCPAN/Server/Model/ESModel.pm b/lib/MetaCPAN/Server/Model/ESModel.pm index e14334052..6f27a4052 100644 --- a/lib/MetaCPAN/Server/Model/ESModel.pm +++ b/lib/MetaCPAN/Server/Model/ESModel.pm @@ -2,7 +2,8 @@ package MetaCPAN::Server::Model::ESModel; use Moose; -use MetaCPAN::Model (); +use MetaCPAN::Model (); +use MetaCPAN::Model::ESWrapper (); extends 'Catalyst::Model'; @@ -16,7 +17,8 @@ has _esx_model => ( lazy => 1, default => sub { my $self = shift; - MetaCPAN::Model->new( es => $self->es ); + my $es = MetaCPAN::Model::ESWrapper->new( $self->es ); + MetaCPAN::Model->new( es => $es ); }, ); diff --git a/t/00_setup.t b/t/00_setup.t index b0341ff21..8d88f8ba7 100644 --- a/t/00_setup.t +++ b/t/00_setup.t @@ -36,9 +36,6 @@ ok( $tmp_dir->stat, "$tmp_dir exists for testing" ); my $server = MetaCPAN::TestServer->new; $server->setup; -# Run the Mappings Tests -$server->test_mappings; - my $config = get_config(); $config->{es} = $server->es_client; diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index 324fc935f..e2ddf311d 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -256,96 +256,5 @@ sub prepare_user_test_data { ); } -sub test_mappings { - my $self = $_[0]; - - $self->test_index_missing; - $self->test_field_mismatch; -} - -sub test_index_missing { - my $self = $_[0]; - - subtest 'missing index' => sub { - my $cover_mapping_json = encode_json( es_config->mapping('cover') ); - - subtest 'delete cover index' => sub { - local @ARGV = qw(mapping --delete_index cover); - my $mapping - = MetaCPAN::Script::Mapping->new_with_options( - $self->_config ); - - ok( $mapping->run, "deletion 'cover' succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - subtest 're-create cover index' => sub { - local @ARGV = ( - 'mapping', '--create_index', - 'cover', '--patch_mapping', - qq({ "cover": $cover_mapping_json }) - ); - my $mapping - = MetaCPAN::Script::Mapping->new_with_options( - $self->_config ); - - ok( $mapping->run, "creation 'cover' succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - }; -} - -sub test_field_mismatch { - my $self = $_[0]; - - subtest 'field mismatch' => sub { - my $sfieldjson = q({ - "properties" : { - "version" : { - "index" : "not_analyzed", - "ignore_above" : 2048, - "type" : "string" - } - } -}); - my $sfieldchangejson = q({ - "properties" : { - "version" : { - "type" : "string", - "index" : "not_analyzed", - "ignore_above" : 1024 - } - } -}); - - subtest 'mapping change field' => sub { - local @ARGV = ( - 'mapping', '--update_index', - 'cover', '--patch_mapping', - qq({ "cover": $sfieldchangejson }) - ); - my $mapping - = MetaCPAN::Script::Mapping->new_with_options( - $self->_config ); - - ok( $mapping->run, "change 'cover' succeeds" ); - - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - subtest 'mapping re-establish field' => sub { - local @ARGV = ( - 'mapping', '--update_index', - 'cover', '--patch_mapping', - qq({ "cover": $sfieldjson }) - ); - my $mapping - = MetaCPAN::Script::Mapping->new_with_options( - $self->_config ); - - ok( $mapping->run, "re-establish 'cover' succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - }; -} - __PACKAGE__->meta->make_immutable; 1; diff --git a/t/script/mapping.t b/t/script/mapping.t deleted file mode 100644 index 89ebfdb18..000000000 --- a/t/script/mapping.t +++ /dev/null @@ -1,73 +0,0 @@ -use strict; -use warnings; -use lib 't/lib'; - -use MetaCPAN::Script::Mapping (); -use MetaCPAN::Server::Config (); -use Test::More; - -my $config = MetaCPAN::Server::Config::config(); - -subtest 'create, delete index' => sub { - subtest 'create index' => sub { - my $smockindexjson = q({ - "mock_index": { - "properties": { - "mock_field": { - "type": "string", - "ignore_above": 2048, - "index": "not_analyzed" - } - } - } - }); - my %args = ( - '--create_index' => 'mock_index', - '--patch_mapping' => $smockindexjson, - ); - local @ARGV = ( 'mapping', %args ); - my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); - - ok( $mapping->run, "creation 'mock_index' succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - subtest 'info shows new index' => sub { - local @ARGV = ( 'mapping', '--show_cluster_info' ); - my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); - - ok( $mapping->run, "show info succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - - ok( defined $mapping->indices_info, 'Index Info built' ); - ok( defined $mapping->indices_info->{'mock_index'}, - 'Created Index printed' ); - }; - subtest 'delete index' => sub { - local @ARGV = ( 'mapping', '--delete_index', 'mock_index' ); - my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); - - ok( $mapping->run, "deletion 'mock_index' succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - }; - subtest 'info does not show deleted index' => sub { - local @ARGV = ( 'mapping', '--show_cluster_info' ); - my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); - - ok( $mapping->run, "show info succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); - - ok( defined $mapping->indices_info, 'Index Info printed' ); - ok( !defined $mapping->indices_info->{'mock_index'}, - 'Deleted Index not printed' ); - }; -}; - -subtest 'mapping verification succeeds' => sub { - local @ARGV = ( 'mapping', '--verify', ); - my $mapping = MetaCPAN::Script::Mapping->new_with_options($config); - - ok( $mapping->run, "verification succeeds" ); - is( $mapping->exit_code, 0, "Exit Code '0' - No Error" ); -}; - -done_testing(); diff --git a/t/server/controller/author.t b/t/server/controller/author.t index 9006bb76e..1baa44bfc 100644 --- a/t/server/controller/author.t +++ b/t/server/controller/author.t @@ -2,7 +2,7 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( app GET POST test_psgi ); +use MetaCPAN::Server::Test qw( app GET POST test_psgi es ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); use Test::More; @@ -56,7 +56,12 @@ test_psgi app, sub { if ( $k eq '/author/_mapping' ) { my ($index) = keys %{$json}; - ok( ref $json->{$index}{mappings}{author} eq 'HASH', '_mapping' ); + my $check_mappings = $json->{$index}{mappings}; + if ( es->api_version le '5_0' ) { + $check_mappings = $check_mappings->{author}; + } + + ok( $check_mappings->{properties}{name}, '_mapping' ); } }