diff --git a/.circleci/config.yml b/.circleci/config.yml index dc0c7ae68..0aa915976 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,64 +15,24 @@ jobs: resource_class: large steps: - run: - name: docker-compose version - command: docker-compose --version + name: docker compose version + command: docker compose version + - checkout - run: + name: create coverage directory command: | - git clone https://github.com/metacpan/metacpan-docker.git - cd metacpan-docker - name: metacpan-docker checkout - - checkout: - path: metacpan-docker/src/metacpan-api + mkdir cover_db + chmod o+w cover_db - run: + name: docker compose build command: | - pushd metacpan-docker - ./bin/metacpan-docker init - name: clone missing repositories + docker compose --profile test build api-test - run: - command: | - pushd metacpan-docker - docker-compose build --build-arg CPM_ARGS='--with-test' api_test - name: compose build - - run: - command: | - pushd metacpan-docker - ./bin/metacpan-docker init - docker-compose --verbose up -d api_test - name: compose up - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cpm install -g Devel::Cover - name: install Devel::Cover - # Since we're running docker-compose -d, we don't actually know if - # Elasticsearch is available at the time this build step begins. We - # probably need to wait for it here, so we'll add our own check. - - run: - command: | - pushd metacpan-docker - ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test - name: wait for ES - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test env HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t name: run tests with coverage + command: | + docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report json' # We are relying on environment variables from the host to be available when # we publish the report, so we publish from the host rather than trying # to propagate env variables to the container. - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cover -report json - name: create coverage report - codecov/upload: - file: metacpan-docker/src/metacpan-api/cover_db/cover.json - - run: - command: | - pushd metacpan-docker - docker-compose logs - docker stats --no-stream - docker ps -a | head - name: docker-compose logs - when: on_fail + file: cover_db/cover.json diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 000000000..817561902 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,46 @@ +name: Build container +on: + push: + branches: + - master + - staging + - prod + pull_request: + types: [opened, synchronize, labeled] + branches: + - master + workflow_dispatch: +jobs: + docker-build: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-container') + runs-on: ubuntu-22.04 + name: Docker Build and Push + steps: + - name: Generate Auth Token + uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: metacpan + - uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - uses: metacpan/metacpan-actions/docker-build-push@master + id: build-push + with: + docker_hub_username: ${{ secrets.DOCKER_HUB_USER }} + docker_hub_password: ${{ secrets.DOCKER_HUB_TOKEN }} + ghcr_username: ${{ github.repository_owner }} + ghcr_password: ${{ secrets.GITHUB_TOKEN }} + test-target: test +# - name: Update deployed image +# if: steps.find-tag-names.outputs.latest +# uses: metacpan/metacpan-actions/update-deployed-tag:master +# with: +# token: ${{ steps.app-token.outputs.token }} +# app: api +# environment: prod +# base-tag: ${{ steps.find-tag-names.outputs.latest }} +# tag: ${{ steps.find-tag-names.outputs.sha }} + diff --git a/.github/workflows/build-deployment-container.yml b/.github/workflows/build-deployment-container.yml deleted file mode 100644 index 721d46dc7..000000000 --- a/.github/workflows/build-deployment-container.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Build deployment container -on: - push: - branches: - - prod - - staging - workflow_dispatch: -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push SHA - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:$GITHUB_SHA - - name: run Perl tests - run: docker run -i metacpan/metacpan-api carton exec prove -lr --jobs 2 t - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:$GITHUB_SHA - diff --git a/.github/workflows/build-production-container.yml b/.github/workflows/build-production-container.yml deleted file mode 100644 index 584f9b54f..000000000 --- a/.github/workflows/build-production-container.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: Build Production Container -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push latest - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:latest - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:latest - if: success() && github.ref == 'refs/heads/master' diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml new file mode 100644 index 000000000..5eb31e5e0 --- /dev/null +++ b/.github/workflows/code-formatting.yml @@ -0,0 +1,52 @@ +--- +name: Code Formatting +on: + push: + branches: + - 'master' + merge_group: + pull_request: + branches: + - '*' + workflow_dispatch: + +jobs: + code-formatting: + runs-on: ubuntu-24.04 + name: Code Formatting + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Fetch base ref + if: ${{ github.event.pull_request }} + run: git fetch origin ${{ github.base_ref }}:upstream + - name: Install Carton + uses: perl-actions/install-with-cpm@v1 + with: + install: Carton + - name: Install CPAN deps + uses: perl-actions/install-with-cpm@v1 + with: + cpanfile: 'cpanfile' + args: > + --resolver=snapshot + --without-runtime + --without-test + --without-build + --with-develop + - name: Install precious + run: ./bin/install-precious /usr/local/bin + env: + GITHUB_TOKEN: ${{ github.token }} + - run: perltidy --version + - name: Select files + id: select-files + run: | + if [[ -n "${{ github.event.pull_request.number }}" ]]; then + echo 'precious-args=--git-diff-from upstream' >> "$GITHUB_OUTPUT" + else + echo 'precious-args=--all' >> "$GITHUB_OUTPUT" + fi + - name: Lint files + run: precious lint ${{ steps.select-files.outputs.precious-args }} diff --git a/.gitignore b/.gitignore index 194aaad50..cd6e79b24 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ /perltidy.LOG /pm_to_blib /var +/bin/omegasort +/bin/precious +/bin/ubi /etc/metacpan_local.pl /t/var/darkpan/ /t/var/log/ diff --git a/Dockerfile b/Dockerfile index 976a9f5f2..2948c9f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,101 @@ -# hadolint ignore=DL3007 -FROM metacpan/metacpan-base:latest - -COPY cpanfile cpanfile.snapshot /metacpan-api/ -WORKDIR /metacpan-api - -# CPM installations of dependencies does not install or run tests. This is -# because the modules themselves have been tested, and the metacpan use of the -# modules is tested by the test suite. Removing the tests, reduces the overall -# size of the images. -RUN mkdir /CPAN \ - && apt-get update \ - && apt-get satisfy -y --no-install-recommends 'rsync (>= 3.2.3)' 'jq (>= 1.6)' \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && cpm install --global \ - && git config --global --add safe.directory /metacpan-api \ - && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/* +ARG SLIM_BUILD +ARG MAYBE_BASE_BUILD=${SLIM_BUILD:+server-base-slim} +ARG BASE_BUILD=${MAYBE_BASE_BUILD:-server-base} + +################### Web Server Base +FROM metacpan/metacpan-base:main-20250531-090128 AS server-base +FROM metacpan/metacpan-base:main-20250531-090129-slim AS server-base-slim + +################### CPAN Prereqs +FROM server-base AS build-cpan-prereqs +SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ] + +WORKDIR /app/ + +COPY cpanfile cpanfile.snapshot ./ +RUN \ + --mount=type=cache,target=/root/.perl-cpm,sharing=private \ +< '2.02'; -requires 'Perl::Critic', '0.140'; -requires 'Perl::Tidy' => '== 20240511'; -requires 'PPI', '1.274'; # Perl::Critic -requires 'PPIx::QuoteLike', '0.022'; # Perl::Critic -requires 'PPIx::Regexp', '0.085'; # Perl::Critic -requires 'String::Format', '1.18'; # Perl::Critic -requires 'Test::Deep'; -requires 'Test::Fatal'; -requires 'Test::Harness', '3.44'; # Contains App::Prove -requires 'Test::More', '1.302190'; -requires 'Test::Perl::Critic', '1.04'; -requires 'Test::RequiresInternet'; -requires 'Test::Routine', '0.012'; -requires 'Test::Vars', '0.015'; +on test => sub { + requires 'CPAN::Faker', '0.011'; + requires 'Devel::Confess'; + requires 'HTTP::Cookies', '6.10'; + requires 'MetaCPAN::Client', '2.029000'; + requires 'Module::Faker', '== 0.017'; + requires 'Module::Faker::Dist', '== 0.017'; + requires 'OrePAN2', '0.48'; + requires 'Test::Deep'; + requires 'Test::Fatal'; + requires 'Test::Harness', '3.44'; # Contains App::Prove + requires 'Test::More', '1.302190'; + requires 'Test::RequiresInternet'; + requires 'Test::Routine', '0.012'; + requires 'Test::Vars', '0.015'; +}; # author requirements -requires 'App::perlimports'; +on develop => sub { + requires 'App::perlimports'; + requires 'Perl::Critic', '0.140'; + requires 'Perl::Tidy' => '== 20240511'; + requires 'PPI', '1.274'; # Perl::Critic + requires 'PPIx::QuoteLike', '0.022'; # Perl::Critic + requires 'PPIx::Regexp', '0.085'; # Perl::Critic + requires 'String::Format', '1.18'; # Perl::Critic + requires 'Devel::Cover'; +}; diff --git a/deploy/build.sh b/deploy/build.sh deleted file mode 100755 index 64013f312..000000000 --- a/deploy/build.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -source "${DEPLOY_DIR}/vars.sh" - -# ## Go to where the docker file is -cd "${DEPLOY_DIR}/.." - -## Pull the latest docker file from docker hub if there is one -docker pull "$DOCKER_HUB_NAME" || true - -## Issue the build command, adding tags (from CONFIG.sh) -docker build --pull --cache-from "$DOCKER_HUB_NAME" --tag $DOCKER_HUB_NAME --tag $VERSION_TAG . diff --git a/deploy/push.sh b/deploy/push.sh deleted file mode 100755 index bb24bd2ff..000000000 --- a/deploy/push.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -source "${DEPLOY_DIR}/vars.sh" - -cd "${DEPLOY_DIR}/.." - -docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWD" - -docker push "$DOCKER_HUB_NAME" \ No newline at end of file diff --git a/deploy/vars.sh b/deploy/vars.sh deleted file mode 100755 index 7431405a6..000000000 --- a/deploy/vars.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -## Edit this -if [ -z $DOCKER_IMAGE_NAME ]; then - echo "DOCKER_IMAGE_NAME is not defined" - exit 1; -fi - -## Should not need to edit this -export DOCKER_HUB_NAME="metacpan/${DOCKER_IMAGE_NAME}" -export VERSION="${TRAVIS_BUILD_NUMBER:-UNKNOWN-BUILD-NUMBER}" -export VERSION_TAG="${DOCKER_HUB_NAME}:${VERSION}" - diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..99066703d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +services: + api-server: + build: + context: . + target: develop + volumes: + - './:/app/' + - '/app/local' + ports: + - '8001:8000' + environment: + # default is 120, shorten to work with compose label + COLUMNS: 96 + develop: + watch: + - path: ./cpanfile + action: rebuild + + api-test: + profiles: + - test + depends_on: + elasticsearch-test: + condition: service_healthy + build: + context: . + target: test + environment: + NET_ASYNC_HTTP_MAXCONNS: 1 + COLUMNS: 80 + ES: http://elasticsearch-test:9200 + HARNESS_ACTIVE: 1 + # Instantiate Catalyst models using metacpan_server_testing.conf + METACPAN_SERVER_CONFIG_LOCAL_SUFFIX: testing + MINICPAN: /CPAN + DEVEL_COVER_OPTIONS: +ignore,^t/|^test-data/|^etc/|^local/ + networks: + - elasticsearch + volumes: + - type: volume + source: elasticsearch-test + target: /usr/share/elasticsearch/data + + elasticsearch-test: + profiles: + - test + platform: linux/amd64 + image: elasticsearch:2.4 + environment: + - discovery.type=single-node + healthcheck: + timeout: 5s + start_period: 60s + test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] + ports: + - "9200" + networks: + - elasticsearch + +networks: + elasticsearch: + +volumes: + elasticsearch-test: diff --git a/git/hooks/pre-commit b/git/hooks/pre-commit index 7b1572230..9256ae257 100755 --- a/git/hooks/pre-commit +++ b/git/hooks/pre-commit @@ -1,13 +1,15 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use Config; -# Hack to use carton's local::lib. -use lib 'local/lib/perl5'; -$ENV{PATH} .= $Config{path_sep}.'local/bin'; -$ENV{PERL5LIB} = join $Config{path_sep}, - grep defined, $ENV{PERL5LIB}, 'local/lib/perl5'; - -use Code::TidyAll::Git::Precommit; -Code::TidyAll::Git::Precommit->check( no_stash => 1 ); +#!/bin/bash + +declare -i status +status=0 + +PRECIOUS=$(which precious) +if [[ -z $PRECIOUS ]]; then + PRECIOUS=./bin/precious +fi + +if ! "$PRECIOUS" lint -s; then + status+=1 +fi + +exit $status diff --git a/improve-search-results/.gitignore b/improve-search-results/.gitignore deleted file mode 100644 index 3374723e1..000000000 --- a/improve-search-results/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/local/ -# carton/local::lib diff --git a/improve-search-results/README.md b/improve-search-results/README.md deleted file mode 100644 index 142d17e2a..000000000 --- a/improve-search-results/README.md +++ /dev/null @@ -1,41 +0,0 @@ - -# Search result comparison system - -## Why - -We want to improve MetaCPAN's search results, getting them at least as good as search.cpan.org's but ideally -even better. - -## How - -Run multiple searches (via the API that the web UI now uses), with different weights (that are now arguments) and compare to each other (so one weighthing doesn't -then break another, or at least we can come to some -balance). - -### Installing - -You will need postgres installed with a database -that matches the current user and the current user needs -access (the MetaCPAN developer vm sets this up for you). - -```sh -cpanm Carton -carton install -``` - -### Running tests - -```sh - -carton exec /opt/perl-5.22.2/bin/perl ./app.pl eval 'app->perform_all_searches' -``` - -### Viewing results site -```sh -carton exec /opt/perl-5.22.2/bin/perl ./app.pl daemon -m production -l http://*:5000 -``` - - - - - diff --git a/improve-search-results/app.pl b/improve-search-results/app.pl deleted file mode 100755 index 6567c6fba..000000000 --- a/improve-search-results/app.pl +++ /dev/null @@ -1,217 +0,0 @@ -use Mojolicious::Lite; - -use Mojo::Pg; -use List::Util qw( first ); - -my $user = getpwuid($<); # for vagrant user on dev box - -# carton exec /opt/perl-5.22.2/bin/perl ./app.pl daemon -m production -l http://*:5000 - -helper pg => sub { state $pg = Mojo::Pg->new("postgresql:///${user}") }; - -app->pg->auto_migrate(1)->migrations->from_data; - -helper insert_search => sub { - my ( $c, $search, $expect ) = @_; - return !!$c->pg->db->query( <<' SQL', $search, $expect )->rows; - insert into searches (search, expect) values (?, ?) - SQL -}; - -helper insert_source => sub { - my ( $c, $name, $query ) = @_; - return !!$c->pg->db->query( <<' SQL', $name, $query )->rows; - insert into sources (name, query) values (?, ?, ?) - SQL -}; - -helper get_results => sub { - my $c = shift; - return $c->pg->db->query(<<' SQL')->expand->hash->{results}; - select json_object_agg(search, results) as results - from ( - select - searches.search, - json_object_agg(sources.name, results.rank) as results - from results - inner join searches on searches.id = results.search_id - inner join sources on sources.id = results.source_id - group by searches.search - ) x - SQL -}; - -helper perform_all_searches => sub { - my ($c) = @_; - my $queries = $c->pg->db->query(<<' SQL'); - select - searches.id as search_id, - sources.id as source_id, - searches.search, - searches.expect, - sources.name, - results.rank - from searches - cross join sources - left join results on searches.id = results.search_id - and sources.id = results.source_id - SQL - my $db = $c->pg->db; - my $sql - = 'insert into results (search_id, source_id, rank) values (?, ?, ?)'; - $queries->hashes->each( sub { - my $query = shift; - return if $query->{rank}; - my $rank = $c->perform_one_search( - @{$query}{qw/search expect name query/} ); - $db->query( $sql, @{$query}{qw/search_id source_id/}, $rank ); - } ); -}; - -helper perform_one_search => sub { - my ( $c, $search, $expect, $name, $query ) = @_; - - my $rank - = $name eq 'SCO' ? _perform_sco( $c, $search, $expect ) - : $name eq 'MWEB' ? _perform_mweb( $c, $search, $expect ) - : _perform_mquery( $c, $search, $expect, $query ); - - return $rank // 100; -}; - -sub _perform_sco { - my ( $c, $search, $expect ) = @_; - my $url = Mojo::URL->new('http://search.cpan.org/search?mode=all&n=100'); - $url->query( [ query => $search ] ); - my $tx = $c->app->ua->get($url); - my $res = $tx->res->dom->find('.sr')->map('all_text')->to_array; - my $idx = first { $res->[$_] eq $expect } @{ $res->to_array }; - return $idx < 0 ? undef : $idx + 1; -} - -sub _perform_mweb { - my ( $c, $search, $expect ) = @_; - my $url = Mojo::URL->new('https://metacpan.org/search?size=100'); - $url->query( [ q => $search ] ); - my $tx = $c->app->ua->get($url); - my $res - = $tx->res->dom->find('.module-result big strong a')->map('all_text') - ->to_array; - my $idx = first { $res->[$_] eq $expect } 0 .. $#{$res}; - return $idx < 0 ? undef : $idx + 1; -} - -sub _perform_mquery { } - -get '/' => 'index'; - -get '/results' => sub { - my $c = shift; - $c->render( json => $c->get_results ); -}; - -app->start; - -__DATA__ - -@@ index.html.ep - -<%== perform_all_searches %> - - - - - - MetaCPAN Search Comparison - - - - - - - - - - -@@ migrations - --- 1 up - -create table searches ( - id bigserial primary key, - search text not null unique, - expect text not null -); -insert into searches (search, expect) values -('tmpfile', 'File::Temp'), -('path', 'Path::Tiny'), -('dbix', 'DBIx::Class'), -('uri', 'URI'); - -create table sources ( - id bigserial primary key, - name text not null unique, - query text -); -insert into sources (name) values ('SCO'), ('MWEB'); - -create table results ( - id bigserial primary key, - search_id bigint references searches on delete cascade, - source_id bigint references sources on delete cascade, - rank integer -); - --- 1 down - -drop table if exists searches cascade; -drop table if exists sources cascade; -drop table if exists results cascade; diff --git a/improve-search-results/cpanfile b/improve-search-results/cpanfile deleted file mode 100644 index a4f1508a4..000000000 --- a/improve-search-results/cpanfile +++ /dev/null @@ -1,6 +0,0 @@ -requires 'perl', '5.010'; - -requires 'Mojolicious', 7.23; -requires 'Mojolicious::Lite', 0; -requires 'Mojo::Pg', 2.35; - diff --git a/improve-search-results/cpanfile.snapshot b/improve-search-results/cpanfile.snapshot deleted file mode 100644 index 2d4bd0476..000000000 --- a/improve-search-results/cpanfile.snapshot +++ /dev/null @@ -1,241 +0,0 @@ -# carton snapshot format: version 1.0 -DISTRIBUTIONS - DBD-Pg-3.5.3 - pathname: T/TU/TURNSTEP/DBD-Pg-3.5.3.tar.gz - provides: - Bundle::DBD::Pg v3.5.3 - DBD::Pg v3.5.3 - requirements: - DBI 1.614 - ExtUtils::MakeMaker 6.11 - Test::More 0.88 - Time::HiRes 0 - version 0 - DBI-1.636 - pathname: T/TI/TIMB/DBI-1.636.tar.gz - provides: - Bundle::DBI 12.008696 - DBD::DBM 0.08 - DBD::DBM::Statement 0.08 - DBD::DBM::Table 0.08 - DBD::DBM::db 0.08 - DBD::DBM::dr 0.08 - DBD::DBM::st 0.08 - DBD::ExampleP 12.014311 - DBD::ExampleP::db 12.014311 - DBD::ExampleP::dr 12.014311 - DBD::ExampleP::st 12.014311 - DBD::File 0.44 - DBD::File::DataSource::File 0.44 - DBD::File::DataSource::Stream 0.44 - DBD::File::Statement 0.44 - DBD::File::Table 0.44 - DBD::File::TableSource::FileSystem 0.44 - DBD::File::db 0.44 - DBD::File::dr 0.44 - DBD::File::st 0.44 - DBD::Gofer 0.015327 - DBD::Gofer::Policy::Base 0.010088 - DBD::Gofer::Policy::classic 0.010088 - DBD::Gofer::Policy::pedantic 0.010088 - DBD::Gofer::Policy::rush 0.010088 - DBD::Gofer::Transport::Base 0.014121 - DBD::Gofer::Transport::corostream undef - DBD::Gofer::Transport::null 0.010088 - DBD::Gofer::Transport::pipeone 0.010088 - DBD::Gofer::Transport::stream 0.014599 - DBD::Gofer::db 0.015327 - DBD::Gofer::dr 0.015327 - DBD::Gofer::st 0.015327 - DBD::NullP 12.014715 - DBD::NullP::db 12.014715 - DBD::NullP::dr 12.014715 - DBD::NullP::st 12.014715 - DBD::Proxy 0.2004 - DBD::Proxy::RPC::PlClient 0.2004 - DBD::Proxy::db 0.2004 - DBD::Proxy::dr 0.2004 - DBD::Proxy::st 0.2004 - DBD::Sponge 12.010003 - DBD::Sponge::db 12.010003 - DBD::Sponge::dr 12.010003 - DBD::Sponge::st 12.010003 - DBDI 12.015129 - DBI 1.636 - DBI::Const::GetInfo::ANSI 2.008697 - DBI::Const::GetInfo::ODBC 2.011374 - DBI::Const::GetInfoReturn 2.008697 - DBI::Const::GetInfoType 2.008697 - DBI::DBD 12.015129 - DBI::DBD::Metadata 2.014214 - DBI::DBD::SqlEngine 0.06 - DBI::DBD::SqlEngine::DataSource 0.06 - DBI::DBD::SqlEngine::Statement 0.06 - DBI::DBD::SqlEngine::Table 0.06 - DBI::DBD::SqlEngine::TableSource 0.06 - DBI::DBD::SqlEngine::TieMeta 0.06 - DBI::DBD::SqlEngine::TieTables 0.06 - DBI::DBD::SqlEngine::db 0.06 - DBI::DBD::SqlEngine::dr 0.06 - DBI::DBD::SqlEngine::st 0.06 - DBI::Gofer::Execute 0.014283 - DBI::Gofer::Request 0.012537 - DBI::Gofer::Response 0.011566 - DBI::Gofer::Serializer::Base 0.009950 - DBI::Gofer::Serializer::DataDumper 0.009950 - DBI::Gofer::Serializer::Storable 0.015586 - DBI::Gofer::Transport::Base 0.012537 - DBI::Gofer::Transport::pipeone 0.012537 - DBI::Gofer::Transport::stream 0.012537 - DBI::Profile 2.015065 - DBI::ProfileData 2.010008 - DBI::ProfileDumper 2.015325 - DBI::ProfileDumper::Apache 2.014121 - DBI::ProfileSubs 0.009396 - DBI::ProxyServer 0.3005 - DBI::ProxyServer::db 0.3005 - DBI::ProxyServer::dr 0.3005 - DBI::ProxyServer::st 0.3005 - DBI::SQL::Nano 1.015544 - DBI::SQL::Nano::Statement_ 1.015544 - DBI::SQL::Nano::Table_ 1.015544 - DBI::Util::CacheMemory 0.010315 - DBI::Util::_accessor 0.009479 - DBI::common 1.636 - requirements: - ExtUtils::MakeMaker 6.48 - Test::Simple 0.90 - perl 5.008 - Mojo-Pg-2.35 - pathname: S/SR/SRI/Mojo-Pg-2.35.tar.gz - provides: - Mojo::Pg 2.35 - Mojo::Pg::Database undef - Mojo::Pg::Migrations undef - Mojo::Pg::PubSub undef - Mojo::Pg::Results undef - Mojo::Pg::Transaction undef - requirements: - DBD::Pg 3.005001 - ExtUtils::MakeMaker 0 - Mojolicious 7.15 - perl 5.010001 - Mojolicious-7.23 - pathname: S/SR/SRI/Mojolicious-7.23.tar.gz - provides: - Mojo undef - Mojo::Asset undef - Mojo::Asset::File undef - Mojo::Asset::Memory undef - Mojo::Base undef - Mojo::ByteStream undef - Mojo::Cache undef - Mojo::Collection undef - Mojo::Content undef - Mojo::Content::MultiPart undef - Mojo::Content::Single undef - Mojo::Cookie undef - Mojo::Cookie::Request undef - Mojo::Cookie::Response undef - Mojo::DOM undef - Mojo::DOM::CSS undef - Mojo::DOM::HTML undef - Mojo::Date undef - Mojo::EventEmitter undef - Mojo::Exception undef - Mojo::File undef - Mojo::Headers undef - Mojo::HelloWorld undef - Mojo::Home undef - Mojo::IOLoop undef - Mojo::IOLoop::Client undef - Mojo::IOLoop::Delay undef - Mojo::IOLoop::Server undef - Mojo::IOLoop::Stream undef - Mojo::IOLoop::Subprocess undef - Mojo::IOLoop::TLS undef - Mojo::JSON undef - Mojo::JSON::Pointer undef - Mojo::Loader undef - Mojo::Log undef - Mojo::Message undef - Mojo::Message::Request undef - Mojo::Message::Response undef - Mojo::Parameters undef - Mojo::Path undef - Mojo::Reactor undef - Mojo::Reactor::EV undef - Mojo::Reactor::Poll undef - Mojo::Server undef - Mojo::Server::CGI undef - Mojo::Server::Daemon undef - Mojo::Server::Hypnotoad undef - Mojo::Server::Morbo undef - Mojo::Server::PSGI undef - Mojo::Server::PSGI::_IO undef - Mojo::Server::Prefork undef - Mojo::Template undef - Mojo::Transaction undef - Mojo::Transaction::HTTP undef - Mojo::Transaction::WebSocket undef - Mojo::URL undef - Mojo::Upload undef - Mojo::UserAgent undef - Mojo::UserAgent::CookieJar undef - Mojo::UserAgent::Proxy undef - Mojo::UserAgent::Server undef - Mojo::UserAgent::Transactor undef - Mojo::Util undef - Mojo::WebSocket undef - Mojolicious 7.23 - Mojolicious::Command undef - Mojolicious::Command::cgi undef - Mojolicious::Command::cpanify undef - Mojolicious::Command::daemon undef - Mojolicious::Command::eval undef - Mojolicious::Command::generate undef - Mojolicious::Command::generate::app undef - Mojolicious::Command::generate::lite_app undef - Mojolicious::Command::generate::makefile undef - Mojolicious::Command::generate::plugin undef - Mojolicious::Command::get undef - Mojolicious::Command::inflate undef - Mojolicious::Command::prefork undef - Mojolicious::Command::psgi undef - Mojolicious::Command::routes undef - Mojolicious::Command::test undef - Mojolicious::Command::version undef - Mojolicious::Commands undef - Mojolicious::Controller undef - Mojolicious::Lite undef - Mojolicious::Plugin undef - Mojolicious::Plugin::Config undef - Mojolicious::Plugin::Config::Sandbox undef - Mojolicious::Plugin::DefaultHelpers undef - Mojolicious::Plugin::EPLRenderer undef - Mojolicious::Plugin::EPRenderer undef - Mojolicious::Plugin::HeaderCondition undef - Mojolicious::Plugin::JSONConfig undef - Mojolicious::Plugin::Mount undef - Mojolicious::Plugin::PODRenderer undef - Mojolicious::Plugin::TagHelpers undef - Mojolicious::Plugins undef - Mojolicious::Renderer undef - Mojolicious::Routes undef - Mojolicious::Routes::Match undef - Mojolicious::Routes::Pattern undef - Mojolicious::Routes::Route undef - Mojolicious::Sessions undef - Mojolicious::Static undef - Mojolicious::Types undef - Mojolicious::Validator undef - Mojolicious::Validator::Validation undef - Test::Mojo undef - ojo undef - requirements: - ExtUtils::MakeMaker 0 - IO::Socket::IP 0.37 - JSON::PP 2.27103 - Pod::Simple 3.09 - Time::Local 1.2 - perl 5.010001 diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index ab7a9f66b..86b29ab84 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -2,34 +2,20 @@ package MetaCPAN::API; =head1 DESCRIPTION -This is the API web server interface. - - # On vagrant VM - ./bin/run morbo bin/api.pl +This is the API Minion server. # Display information on jobs in queue ./bin/run bin/api.pl minion job -s -To run the api web server, run the following on one of the servers: - - # Run the daemon on a local port (tunnel to display on your browser) - ./bin/run bin/api.pl daemon - =cut use Mojo::Base 'Mojolicious'; -use File::Temp (); -use List::Util qw( any ); -use MetaCPAN::Script::Runner (); -use Try::Tiny qw( catch try ); -use MetaCPAN::Server::Config (); -use MetaCPAN::Types::TypeTiny qw( ES ); - -has es => sub { - ES->assert_coerce( - MetaCPAN::Server::Config::config()->{elasticsearch_servers} ); -}; +use File::Temp (); +use List::Util qw( any ); +use MetaCPAN::Script::Runner (); +use Try::Tiny qw( catch try ); +use MetaCPAN::Server::Config (); sub startup { my $self = shift; @@ -55,7 +41,6 @@ sub startup { $self->minion->add_task( index_favorite => $self->_gen_index_task_sub('favorite') ); - $self->plugin('MetaCPAN::API::Plugin::Model'); $self->_set_up_routes; } @@ -116,36 +101,7 @@ sub _set_up_routes { ); $self->_set_up_oauth_routes; - - $admin->get('home')->to('admin#home')->name('admin-home'); - $admin->post('enqueue')->to('queue#enqueue')->name('enqueue'); - $admin->post('search-identities')->to('admin#search_identities') - ->name('search-identities'); - $admin->get('index-release')->to('queue#index_release') - ->name('index-release'); - $admin->get('identity-search-form')->to('admin#identity_search_form') - ->name('identity_search_form'); - $self->plugin( 'Minion::Admin' => { route => $admin->any('/minion') } ); - $self->plugin( - 'OpenAPI' => { url => $self->home->rel_file('root/static/v1.yml') } ); - -# This route is for when nginx gets updated to no longer strip the `/v1` path. -# By retaining the `/v1` path the OpenAPI spec is picked up and passed -# through Mojolicous. The `rewrite` parameter is stripping the `/v1` before -# it is passed to Catalyst allowing the previous logic to be followed. - $self->plugin( - MountPSGI => { - '/v1' => $self->home->child('app.psgi')->to_string, - rewrite => 1 - } - ); - -# XXX Catch cases when `v1` has been stripped by nginx until migration is complete -# XXX then this path can be removed. - $self->plugin( - MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); - } sub _is_admin { @@ -205,43 +161,12 @@ sub _set_up_oauth_routes { $c->session( is_logged_in => 1 ); $c->session( github_username => $username ); if ( $self->_is_admin($username) ) { - $c->session( github_username => $username ); $c->redirect_to('/admin'); return; } $c->redirect_to( $self->config->{front_end_url} ); }, ); - - $self->plugin( - 'Web::Auth', - module => 'Google', - key => $oauth->{google}->{key}, - secret => $oauth->{google}->{secret}, - user_info => 1, - on_finished => sub { - my ( $c, $access_token, $account_info ) = @_; - my $username = $account_info->{login}; - $c->session( is_logged_in => 1 ); - $c->session( google_username => $username ); - $c->redirect_to( $self->config->{front_end_url} ); - }, - ); - - $self->plugin( - 'Web::Auth', - module => 'Twitter', - key => $oauth->{twitter}->{key}, - secret => $oauth->{twitter}->{secret}, - user_info => 1, - on_finished => sub { - my ( $c, $access_token, $access_secret, $account_info ) = @_; - my $username = $account_info->{screen_name}; - $c->session( is_logged_in => 1 ); - $c->session( twitter_username => $username ); - $c->redirect_to( $self->config->{front_end_url} ); - }, - ); } 1; diff --git a/lib/MetaCPAN/API/Controller/Cover.pm b/lib/MetaCPAN/API/Controller/Cover.pm deleted file mode 100644 index 7479b5154..000000000 --- a/lib/MetaCPAN/API/Controller/Cover.pm +++ /dev/null @@ -1,16 +0,0 @@ -package MetaCPAN::API::Controller::Cover; - -use Mojo::Base 'Mojolicious::Controller'; - -sub lookup { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $results = $c->model->cover->find_release_coverage( $args->{name} ); - return $c->render( openapi => $results ) if $results; - $c->rendered(404); -} - -1; - diff --git a/lib/MetaCPAN/API/Controller/Queue.pm b/lib/MetaCPAN/API/Controller/Queue.pm deleted file mode 100644 index d35c66167..000000000 --- a/lib/MetaCPAN/API/Controller/Queue.pm +++ /dev/null @@ -1,19 +0,0 @@ -package MetaCPAN::API::Controller::Queue; - -use Mojo::Base 'Mojolicious::Controller'; - -my $rel - = 'https://cpan.metacpan.org/authors/id/O/OA/OALDERS/HTML-Restrict-2.2.2.tar.gz'; - -sub enqueue { - my $self = shift; - $self->minion->enqueue( index_release => [ '--latest', $rel ] ); - $self->render( text => 'OK' ); -} - -sub index_release { - my $self = shift; - $self->render( text => 'ok' ); -} - -1; diff --git a/lib/MetaCPAN/API/Controller/Search.pm b/lib/MetaCPAN/API/Controller/Search.pm deleted file mode 100644 index fbc21bdcc..000000000 --- a/lib/MetaCPAN/API/Controller/Search.pm +++ /dev/null @@ -1,32 +0,0 @@ -package MetaCPAN::API::Controller::Search; - -use Mojo::Base 'Mojolicious::Controller'; - -sub first { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $results = $c->model->search->search_for_first_result( $args->{q} ); - return $c->render( openapi => $results ) if $results; - $c->rendered(404); -} - -sub web { - my $c = shift; - return unless $c->openapi->valid_input; - my $args = $c->validation->output; - - my $query = $args->{q}; - my $size = $args->{page_size} // $args->{size} // 20; - my $page = $args->{page} // ( 1 + int( ( $args->{from} // 0 ) / $size ) ); - my $collapsed = $args->{collapsed}; - - my $results - = $c->model->search->search_web( $query, $page, $size, $collapsed ); - - return $c->render( json => $results ); -} - -1; - diff --git a/lib/MetaCPAN/API/Model/Cover.pm b/lib/MetaCPAN/API/Model/Cover.pm deleted file mode 100644 index 243ccae7d..000000000 --- a/lib/MetaCPAN/API/Model/Cover.pm +++ /dev/null @@ -1,33 +0,0 @@ -package MetaCPAN::API::Model::Cover; - -use MetaCPAN::ESConfig qw( es_doc_path ); -use MetaCPAN::Moose; - -use MetaCPAN::Util qw(hit_total); - -with 'MetaCPAN::API::Model::Role::ES'; - -sub find_release_coverage { - my ( $self, $release ) = @_; - - my $query = +{ term => { release => $release } }; - - my $res = $self->_run_query( - es_doc_path('cover'), - body => { - query => $query, - size => 999, - } - ); - hit_total($res) or return {}; - - return +{ - %{ $res->{hits}{hits}[0]{_source} }, - url => "http://cpancover.com/latest/$release/index.html", - }; -} - -__PACKAGE__->meta->make_immutable; - -1; - diff --git a/lib/MetaCPAN/API/Model/Download.pm b/lib/MetaCPAN/API/Model/Download.pm deleted file mode 100644 index 7900b8bf3..000000000 --- a/lib/MetaCPAN/API/Model/Download.pm +++ /dev/null @@ -1,10 +0,0 @@ -package MetaCPAN::API::Model::Download; - -use MetaCPAN::Moose; - -with 'MetaCPAN::API::Model::Role::ES'; - -__PACKAGE__->meta->make_immutable; - -1; - diff --git a/lib/MetaCPAN/Model/ESWrapper.pm b/lib/MetaCPAN/Model/ESWrapper.pm index 9cc17b497..cd990c897 100644 --- a/lib/MetaCPAN/Model/ESWrapper.pm +++ b/lib/MetaCPAN/Model/ESWrapper.pm @@ -2,7 +2,7 @@ package MetaCPAN::Model::ESWrapper; use strict; use warnings; -use MetaCPAN::Types::TypeTiny qw(ES); +use MetaCPAN::Types::TypeTiny qw( ES ); sub new { my ( $class, $es ) = @_; diff --git a/lib/MetaCPAN/Role/Logger.pm b/lib/MetaCPAN/Role/Logger.pm index 258ccfa0a..95d80d662 100644 --- a/lib/MetaCPAN/Role/Logger.pm +++ b/lib/MetaCPAN/Role/Logger.pm @@ -6,7 +6,7 @@ use Moose::Role; use Log::Contextual qw( set_logger ); use Log::Log4perl ':easy'; use MetaCPAN::Types::TypeTiny qw( Logger Str ); -use MooseX::Getopt (); +use MooseX::Getopt (); ## no perlimports use Path::Tiny qw( path ); has level => ( diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index 801b05a39..128c78b76 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -2,11 +2,10 @@ package MetaCPAN::Script::Mapping; use Moose; -use Cpanel::JSON::XS qw( decode_json ); -use DateTime (); +use Cpanel::JSON::XS (); use Log::Contextual qw( :log ); use MetaCPAN::ESConfig qw( es_config ); -use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int Str ); +use MetaCPAN::Types::TypeTiny qw( Bool HashRef Int ); use Time::HiRes qw( sleep time ); use constant { @@ -162,7 +161,7 @@ sub show_info { 'cluster_info' => \%{ $self->cluster_info }, 'indices_info' => \%{ $self->indices_info }, }; - log_info { JSON->new->utf8->pretty->encode($info_rs) }; + log_info { Cpanel::JSON::XS->new->utf8->pretty->encode($info_rs) }; } sub _build_index_config { diff --git a/lib/MetaCPAN/Server/Config.pm b/lib/MetaCPAN/Server/Config.pm index 4bf51f97a..a8b032c7d 100644 --- a/lib/MetaCPAN/Server/Config.pm +++ b/lib/MetaCPAN/Server/Config.pm @@ -36,7 +36,28 @@ sub _zomg { my $v = Data::Visitor::Callback->new( plain_value => sub { return unless defined $_; - s{__HOME__}{$root}ge; + s{ + (__HOME__) + | + (\$\{([^\}]+)\}) + }{ + defined $1 ? $root + : defined $2 ? do { + my $var = $3; + if ($var =~ s{:-(.*)}{}) { + my $sub = $1; + $ENV{$var} // $1; + } + elsif ($var =~ s{:\+(.*)}{}) { + my $sub = $1; + $ENV{$var} ? $sub : ''; + } + else { + $ENV{$var} // ''; + } + } + : '' + }gex; } ); $v->visit($c); diff --git a/lib/MetaCPAN/Server/Controller/Root.pm b/lib/MetaCPAN/Server/Controller/Root.pm index d0fb311bf..a7f0d4fe9 100644 --- a/lib/MetaCPAN/Server/Controller/Root.pm +++ b/lib/MetaCPAN/Server/Controller/Root.pm @@ -4,6 +4,7 @@ use strict; use warnings; use Moose; +use MetaCPAN::Util qw( true ); BEGIN { extends 'MetaCPAN::Server::Controller' } @@ -68,6 +69,12 @@ sub robots : Path("robots.txt") Args(0) { $c->res->body("User-agent: *\nDisallow: /\n"); } +sub healthcheck : Local Args(0) { + my ( $self, $c ) = @_; + $c->stash( { success => true } ); + $c->forward( $c->view('JSON') ); +} + sub end : ActionClass('RenderView') { my ( $self, $c ) = @_; if ( $c->controller->does('MetaCPAN::Server::Role::JSONP') @@ -99,7 +106,6 @@ sub end : ActionClass('RenderView') { # cdn cache time $c->cdn_never_cache(1); } - } 1; diff --git a/lib/MetaCPAN/Types/Internal.pm b/lib/MetaCPAN/Types/Internal.pm index 3bffb47f7..b991f8fe3 100644 --- a/lib/MetaCPAN/Types/Internal.pm +++ b/lib/MetaCPAN/Types/Internal.pm @@ -7,7 +7,7 @@ use ElasticSearchX::Model::Document::Mapping (); use ElasticSearchX::Model::Document::Types qw( Type ); use MetaCPAN::Util qw( is_bool true false ); use MooseX::Getopt::OptionTypeMap (); -use MooseX::Types::Moose qw( Item Any Bool ArrayRef HashRef ); +use MooseX::Types::Moose qw( ArrayRef Bool HashRef Item ); use MooseX::Types -declare => [ qw( ESBool @@ -24,7 +24,7 @@ coerce Module, from ArrayRef, via { @$_ ]; }; coerce Module, from HashRef, via { - require MetaCPAN::Document::Module; + require MetaCPAN::Document::Module; ## no perlimports [ MetaCPAN::Document::Module->new($_) ]; }; @@ -40,7 +40,7 @@ coerce Identity, from ArrayRef, via { ]; }; coerce Identity, from HashRef, via { - require MetaCPAN::Model::User::Identity; + require MetaCPAN::Model::User::Identity; ## no perlimports [ MetaCPAN::Model::User::Identity->new($_) ]; }; @@ -56,7 +56,7 @@ coerce Dependency, from ArrayRef, via { ]; }; coerce Dependency, from HashRef, via { - require MetaCPAN::Document::Dependency; + require MetaCPAN::Document::Dependency; ## no perlimports [ MetaCPAN::Document::Dependency->new($_) ]; }; @@ -72,7 +72,9 @@ coerce Profile, from ArrayRef, via { ]; }; coerce Profile, from HashRef, via { + ## no perlimports require MetaCPAN::Document::Author::Profile; + ## use perlimports [ MetaCPAN::Document::Author::Profile->new($_) ]; }; diff --git a/metacpan_server_testing.yaml b/metacpan_server_testing.yaml index f47e101d5..23b6e4406 100644 --- a/metacpan_server_testing.yaml +++ b/metacpan_server_testing.yaml @@ -8,7 +8,7 @@ source_base: var/t/tmp/source elasticsearch_servers: client: '2_0::Direct' - nodes: http://elasticsearch_test:9200 + nodes: ${ES:-http://elasticsearch_test:9200} minion_dsn: "postgresql://metacpan:t00lchain@pghost:5432/minion_queue" diff --git a/precious.toml b/precious.toml index 54034f059..21be953fd 100644 --- a/precious.toml +++ b/precious.toml @@ -6,7 +6,7 @@ excludes = [ [commands.perlimports] type = "both" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perlimports" ] lint-flags = ["--lint" ] tidy-flags = ["-i" ] @@ -15,14 +15,14 @@ expect-stderr = true [commands.perlcritic] type = "lint" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perlcritic", "--profile=$PRECIOUS_ROOT/.perlcriticrc" ] ok-exit-codes = 0 lint-failure-exit-codes = 2 [commands.perltidy] type = "both" -include = [ "**/*.{pl,pm,t,psgi}" ] +include = [ "**/*.{pl,pm,t,psgi}", "bin/metacpan" ] cmd = [ "perltidy", "--profile=$PRECIOUS_ROOT/.perltidyrc" ] lint-flags = [ "--assert-tidy", "--no-standard-output", "--outfile=/dev/null" ] tidy-flags = [ "--backup-and-modify-in-place", "--backup-file-extension=/" ] diff --git a/root/static/definitions/common.yml b/root/static/definitions/common.yml deleted file mode 100644 index 4835b3059..000000000 --- a/root/static/definitions/common.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- - -ErrorModel: - type: "object" - required: - - "code" - - "message" - properties: - code: - type: "integer" - format: "int32" - message: - type: "string" diff --git a/root/static/definitions/definitions.yml b/root/static/definitions/definitions.yml deleted file mode 100644 index b292922fa..000000000 --- a/root/static/definitions/definitions.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -# Maintain names and descriptions of common attributes -dist_fav_count: - description: Number of times favorited diff --git a/root/static/definitions/results.yml b/root/static/definitions/results.yml deleted file mode 100644 index eaf7183f1..000000000 --- a/root/static/definitions/results.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- - -search_result_items: - type: object - properties: - distribution: - type: string - hits: - title: Hits - type: array - items: - $ref: "#/search_result_hit" - - total: - type: integer -search_result_hit: - type: object - properties: - description: - type: string - documentation: - type: string - authorized: - type: boolean - path: - type: string - author: - type: string - id: - type: string - date: - type: string - favorites: - type: - - "integer" - - "null" - status: - type: string - score: - type: number - module: - type: array - items: - type: object - properties: - associated_pod: - type: string - indexed: - type: boolean - name: - type: string - authorized: - type: boolean - version_numified: - type: number - distribution: - type: string - indexed: - type: boolean - pod_lines: - type: array - abstract: - type: string - release: - type: string -dependency: - type: object - properties: - module: - type: string - # "Mojolicious", - phase: - type: string - # "runtime", - version: - type: string - # "8", - relationship: - type: string - # "requires" diff --git a/root/static/index.html b/root/static/index.html deleted file mode 100644 index 3c119143f..000000000 --- a/root/static/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - MetaCPAN API - - - - - - - - - - - - - diff --git a/root/static/requests/cover.yml b/root/static/requests/cover.yml deleted file mode 100644 index 639e96730..000000000 --- a/root/static/requests/cover.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- - -cover: - get: - tags: - - Coverage - operationId: cover - x-mojo-to: Cover#lookup - summary: Get coverage details about a release - parameters: - - name: name - in: path - description: | - The name of the Release - type: string - required: true - # tell Mojolicious to use relaxed placeholders when - # parsing this parameter (dots are allowed) - x-mojo-placeholder: "#" - responses: - 200: - description: Release response - schema: - type: object - properties: - url: - type: string - description: URL for cpancover report - version: - type: string - description: Package version string - release: - type: string - description: Package name with version - criteria: - type: object - description: CPAN Cover results - properties: - total: - type: string - description: Percentage of total code coverage - condition: - type: string - description: Percentage of condition code coverage - statement: - type: string - description: Percentage of statement code coverage - branch: - type: string - description: Percentage of branch code coverage - subroutine: - type: string - description: Percentage of subroutine code coverage - distribution: - type: string - description: Name of the distribution - default: - description: "unexpected error" - schema: - $ref: "../definitions/common.yml#/ErrorModel" diff --git a/root/static/requests/release.yml b/root/static/requests/release.yml deleted file mode 100644 index a872d0364..000000000 --- a/root/static/requests/release.yml +++ /dev/null @@ -1,48 +0,0 @@ - /release/recent: - get: - tags: - - Release - operationId: release_recent - x-mojo-to: Release#recent - summary: Get recent releases - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string - dependency: - type: array - items: - $ref: "./definitions/results.yml#/dependency" - /release/{name}: - get: - tags: - - Release - operationId: release_by_name - x-mojo-to: Release#by_name - summary: Get details about a release - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string diff --git a/root/static/requests/search.yml b/root/static/requests/search.yml deleted file mode 100644 index 60f6824af..000000000 --- a/root/static/requests/search.yml +++ /dev/null @@ -1,124 +0,0 @@ ---- - -search_web: - get: - tags: - - Search - operationId: search_web - x-mojo-to: Search#web - summary: Perform API search in the same fashion as the Web UI - parameters: - - name: q - in: query - description: | - The query search term. If the search term contains a term with the - tags `dist:` or `module:` results will be in expanded form, otherwise - collapsed form. - - See also `collapsed` - type: string - required: true - - name: page - in: query - description: The page of the results to return - type: integer - - name: page_size - in: query - description: Number of results per page - type: integer - - name: from - in: query - description: | - The offset to use in the result set. Deprecated. Only used if `page` - is not set. - type: integer - default: 0 - - name: size - in: query - description: | - Number of results per page. Deprecated. Only used if `page_size` is - not set. - type: integer - default: 20 - - name: collapsed - in: query - description: | - Force a collapsed even when searching for a particular - distribution or module name. - type: boolean - responses: - 200: - description: Search response - schema: - type: object - properties: - total: - type: integer - took: - type: number - collapsed: - type: boolean - results: - title: Results - type: array - items: - $ref: "../definitions/results.yml#/search_result_items" -search_first: - get: - tags: - - Search - operationId: search_for_first - x-mojo-to: Search#first - summary: Perform API search and return the first result (I'm Feeling Lucky) - parameters: - - name: q - in: query - description: | - The query search term. - type: string - required: true - responses: - 200: - description: Search response - schema: - type: object - properties: - path: - type: string - description: Relative path to module with full name - authorized: - type: boolean - description: - type: string - description: Module description - id: - type: string - distribution: - type: string - description: Name of the distribution the module is contained in - author: - type: string - description: Module author ID - release: - type: string - description: Package name with version - status: - type: string - abstract.analyzed: - type: string - description: The module's abstract as analyzed from POD - dist_fav_count: - type: integer - description: Number of times favorited - date: - type: string - description: date module was indexed - documentation: - type: string - pod_lines: - type: array - items: - type: integer - indexed: - type: boolean - description: Is the module indexed by PAUSE diff --git a/root/static/v1.yml b/root/static/v1.yml deleted file mode 100644 index 59c0f3ff1..000000000 --- a/root/static/v1.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -swagger: "2.0" -info: - version: "1.0.0" - title: "MetaCPAN API" -basePath: "/v1" -tags: - - name: Search - description: MetaCPAN Search Endpoints - # - name: Release - # description: Distribution Release Endpoints -schemes: - - "http" - - "https" -consumes: - - "application/json" -produces: - - "application/json" -paths: - /search/web: - $ref: "requests/search.yml#/search_web" - /search/first: - $ref: "requests/search.yml#/search_first" - /cover/{name}: - $ref: "requests/cover.yml#/cover" diff --git a/t/api/controller/admin.t b/t/api/controller/admin.t deleted file mode 100644 index d4921ac7b..000000000 --- a/t/api/controller/admin.t +++ /dev/null @@ -1,39 +0,0 @@ -use strict; -use warnings; -use lib 't/lib'; - -use Test::Mojo; -use Test::More; - -local $ENV{MOJO_SECRET} = 'Magritte'; -local $ENV{GITHUB_KEY} = 'foo'; -local $ENV{GITHUB_SECRET} = 'bar'; - -subtest 'authentication enabled' => sub { - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->post_ok('/admin/enqueue'); - $t->header_is( Location => '/auth/github/authenticate' ); - $t->status_is(302); -}; - -subtest 'index release' => sub { - local $ENV{FORCE_ADMIN_AUTH} = 'tester'; - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->get_ok('/admin/index-release'); - $t->status_is(200); -}; - -subtest 'search identities' => sub { - local $ENV{FORCE_ADMIN_AUTH} = 'tester'; - my $t = Test::Mojo->new('MetaCPAN::API'); - $t->get_ok('/admin/identity-search-form'); - $t->status_is(200); - - $t->post_ok( '/admin/search-identities' => form => - { name => 'pause', key => 'MO' } ); - $t->content_like(qr/\bMO\b/); - $t->content_like(qr/\bpause\b/); - $t->status_is(200); -}; - -done_testing(); diff --git a/t/api/controller/cover.t b/t/api/controller/cover.t deleted file mode 100644 index f8933777e..000000000 --- a/t/api/controller/cover.t +++ /dev/null @@ -1,56 +0,0 @@ -use Mojo::Base -strict; - -use lib 't/lib'; - -use MetaCPAN::TestServer (); -use Test::Mojo (); -use Test::More; - -my $server = MetaCPAN::TestServer->new; - -my $t = Test::Mojo->new( - 'MetaCPAN::API' => { - es => $server->es_client, - secret => 'just a test', - } -); - -my %expect = ( - 'MetaFile-Both-1.1' => { - criteria => { - branch => '12.50', - condition => '0.00', - statement => '63.64', - subroutine => '71.43', - total => '46.51', - }, - distribution => 'MetaFile-Both', - release => 'MetaFile-Both-1.1', - url => 'http://cpancover.com/latest/MetaFile-Both-1.1/index.html', - version => '1.1', - }, - 'Pod-With-Generator-1' => { - criteria => { - branch => '78.95', - condition => '46.67', - statement => '95.06', - subroutine => '100.00', - total => '86.58', - }, - distribution => 'Pod-With-Generator', - release => 'Pod-With-Generator-1', - url => 'http://cpancover.com/latest/Pod-With-Generator-1/index.html', - version => '1', - }, -); - -for my $release ( keys %expect ) { - my $expected = $expect{$release}; - subtest "Check $release" => sub { - - $t->get_ok("/v1/cover/$release")->status_is(200)->json_is($expected) - ->or( sub { diag $t->tx->res->dom } ); - - }; -} -done_testing; diff --git a/t/api/controller/search/first.t b/t/api/controller/search/first.t deleted file mode 100644 index 603182832..000000000 --- a/t/api/controller/search/first.t +++ /dev/null @@ -1,14 +0,0 @@ -use Mojo::Base -strict; - -use Test::Mojo (); -use Test::More; - -my $t = Test::Mojo->new('MetaCPAN::API'); - -$t->get_ok( '/v1/search/first', form => { q => 'Versions::PkgVar' } ) - ->status_is(200)->json_like( '/release' => qr/Versions-(?:\d+)/ ); - -$t->get_ok( '/v1/search/first', form => { q => 'DOESNOTEXISTS' } ) - ->status_is(404)->content_is(''); - -done_testing; diff --git a/t/api/controller/search/web.t b/t/api/controller/search/web.t deleted file mode 100644 index 377b7fae0..000000000 --- a/t/api/controller/search/web.t +++ /dev/null @@ -1,73 +0,0 @@ -use Mojo::Base -strict; - -use Mojo::JSON qw( false true ); -use Test::Mojo; -use Test::More; - -# Note: we need a release marked as status => latest -# so we're using Versions::PkgVar for now -# perhaps it should be smarter later and find one to try? - -my $t = Test::Mojo->new('MetaCPAN::API'); - -subtest 'collapsed' => sub { - $t->get_ok( '/v1/search/web', form => { q => 'Versions::PkgVar' } ) - ->status_is(200)->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'module:Versions::PkgVar' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'module:Versions::PkgVar', collapsed => 1 } ) - ->status_is(200)->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'dist:Versions' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'dist:Versions', collapsed => 1 } )->status_is(200) - ->json_is( '/collapsed' => true ); - - $t->get_ok( '/v1/search/web', form => { q => 'distribution:Versions' } ) - ->status_is(200)->json_is( '/collapsed' => false ); - - $t->get_ok( '/v1/search/web', - form => { q => 'distribution:Versions', collapsed => 1 } ) - ->status_is(200)->json_is( '/collapsed' => true ); -}; - -subtest 'paging' => sub { - my $q = 'this'; - $t->get_ok( '/v1/search/web', form => { q => $q } )->status_is(200); - my $json = $t->tx->res->json; - my $total = $json->{total}; - - if ( $total <= 1 ) { - cmp_ok @{ $json->{results} }, '==', $total, - 'results agree with total'; - diag "Only one search result, skipping remaining paging tests\n"; - return; - } - - diag "Testing paging with $total results\n"; - cmp_ok @{ $json->{results} }, '<=', $total, 'results agree with total'; - - # shrink the page size to one, test limit - $t->get_ok( '/v1/search/web', form => { q => $q, size => 1 } ) - ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); - $json = $t->tx->res->json; - cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; - my $first = $json->{results}[0]{hits}[0]{id}; - - # keep the page size as one, test offset - $t->get_ok( '/v1/search/web', form => { q => $q, size => 1, from => 1 } ) - ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); - $json = $t->tx->res->json; - cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; - my $next = $json->{results}[0]{hits}[0]{id}; - - isnt $first, $next, 'got a different result'; -}; - -done_testing; - diff --git a/t/api/queue.t b/t/api/queue.t index b70e051e9..492244b38 100644 --- a/t/api/queue.t +++ b/t/api/queue.t @@ -2,10 +2,10 @@ use strict; use warnings; use lib 't/lib'; +use Test::More skip_all => 'disabling Minion tests to avoid needing postgres'; use MetaCPAN::DarkPAN (); use Path::Tiny qw( path ); use Test::Mojo; -use Test::More; my $t = Test::Mojo->new('MetaCPAN::API'); my $app = $t->app; diff --git a/t/fff_tidyall.t b/t/fff_tidyall.t deleted file mode 100644 index e96e8e278..000000000 --- a/t/fff_tidyall.t +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use lib 't/lib'; - -use Test::Code::TidyAll qw( tidyall_ok ); -tidyall_ok( verbose => $ENV{TEST_VERBOSE} ); diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index f37309960..da329b7a1 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -2,7 +2,6 @@ package MetaCPAN::TestServer; use MetaCPAN::Moose; -use Cpanel::JSON::XS qw( encode_json ); use MetaCPAN::ESConfig qw( es_config ); use MetaCPAN::Script::Author (); use MetaCPAN::Script::Cover (); diff --git a/t/server/controller/author.t b/t/server/controller/author.t index 7b5538ad0..5abe58e00 100644 --- a/t/server/controller/author.t +++ b/t/server/controller/author.t @@ -2,7 +2,7 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Server::Test qw( app GET POST test_psgi es ); +use MetaCPAN::Server::Test qw( app es GET POST test_psgi ); use MetaCPAN::TestHelpers qw( decode_json_ok test_cache_headers ); use Test::More; diff --git a/templates/admin/identity_search_form.html.ep b/templates/admin/identity_search_form.html.ep deleted file mode 100644 index 43fbf5e77..000000000 --- a/templates/admin/identity_search_form.html.ep +++ /dev/null @@ -1,13 +0,0 @@ -
- - - - Identity value: - - -
diff --git a/templates/admin/index.html.ep b/templates/admin/index.html.ep deleted file mode 100644 index a39edf7f1..000000000 --- a/templates/admin/index.html.ep +++ /dev/null @@ -1,8 +0,0 @@ - diff --git a/templates/admin/search_identities.html.ep b/templates/admin/search_identities.html.ep deleted file mode 100644 index 3428abc44..000000000 --- a/templates/admin/search_identities.html.ep +++ /dev/null @@ -1,3 +0,0 @@ -display results below: -<%= stash('user_data')->{identity}[0]{name} %> -<%= stash('user_data')->{identity}[0]{key} %> diff --git a/templates/layouts/default.html.ep b/templates/layouts/default.html.ep deleted file mode 100644 index fbf9c181c..000000000 --- a/templates/layouts/default.html.ep +++ /dev/null @@ -1,5 +0,0 @@ -

-
MetaCPAN Admin
-

- -<%= content %> diff --git a/templates/queue/index_release.html.ep b/templates/queue/index_release.html.ep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tidyall.ini b/tidyall.ini deleted file mode 100644 index b29f471e9..000000000 --- a/tidyall.ini +++ /dev/null @@ -1,15 +0,0 @@ -jobs = 5 - -[PerlTidy] -select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} -select = bin/{metacpan,wait-for-open} -select = app.psgi -ignore = t/var/**/* -argv = --profile=$ROOT/.perltidyrc - -[PerlCritic] -select = {bin,lib,t,xt}/**/*.{pl,pm,t,psgi} -select = bin/{metacpan,wait-for-open} - -[UniqueLines] -select = .gitignore