Initial rebase of AMO (tag:2019.02.28) to ATN master:

* Don't generate Firefox/Android addons for local dev
* Set Thunderbird as the default app
* Only enable Firefox/Android if in tests
* Add auth bypass for development
* Adjusted legacy extension upload message for ATN
* Add contextmanager function to hack around test fixtures and addons-linter woes
* Update ci workflow
* Lots of test fixes

Author: MelissaAutumn

.github/workflows/ci-tests.yml

@@ -18,32 +18,29 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Start DB
-      run: |
-        docker run --add-host host.docker.internal:host-gateway --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=verysecurepw -e MYSQL_DATABASE=test_olympia -d mysql:8.0 --default-authentication-plugin=mysql_native_password
-        sleep 15
-        docker exec mysql mysql -P 3306 -u root -p"verysecurepw" -e "CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH mysql_native_password BY 'verysecurepw';"
-        docker exec mysql mysql -P 3306 -u root -p"verysecurepw" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1';"
-
-    - name: Start ES
-      run: docker run --add-host host.docker.internal:host-gateway --name es -p 9200:9200 -p 9300:9300 -e ES_NETWORK_HOST=0.0.0.0 -e discovery.type=single-node -d elasticsearch:5.6.12-alpine
-
-    - name: Run docker container
-      run: docker run --add-host host.docker.internal:host-gateway -it -d --name atn -e DATABASES_DEFAULT_URL=mysql://root:verysecurepw@host.docker.internal:3306/test_olympia -e ES_HOSTS=host.docker.internal:9200 -e ELASTICSEARCH_LOCATION=host.docker.internal:9200 -v $GITHUB_WORKSPACE/src:/data/olympia/src:rw thunderbirdops/addons-server bash
-
-    - name: Install deps
-      run: docker exec atn bash -c "make -f Makefile-docker update_deps"
-    
-    - name: Patch Django
-      run:  docker exec atn bash -c "sed -i 's/TX_ISOLATION = /transaction_isolation = /' /usr/local/lib/python2.7/site-packages/django/db/backends/mysql/base.py"
-
-    - name: Run tests
-      run: docker exec atn bash -c 'python -m pytest --junitxml=src/test_report.xml -m "not es_tests" src/olympia/'
-
-    - name: Publish test results
-      uses: EnricoMi/publish-unit-test-result-action@v2
-      if: always()
-      with:
-        files: src/test_report.xml
+      - uses: actions/checkout@v4
+
+      - name: Build Docker Compose
+        run: docker compose build
+
+      - name: Run Docker Compose
+        run: docker compose up -d
+
+      - name: Wait a bit for MySQL to boot
+        run: sleep 10
+
+      - name: Install deps
+        run: docker exec atn bash -c "make -f Makefile-docker update_deps"
+
+      - name: Patch Django
+        run: docker exec atn bash -c "sed -i 's/TX_ISOLATION = /transaction_isolation = /' /usr/local/lib/python3.6/site-packages/django/db/backends/mysql/base.py"
+
+      - name: Run tests
+        run: docker exec atn bash -c 'python -m pytest --junitxml=src/test_report36.xml -m "not es_tests" src/olympia/'
+
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          check_name: "Python 3.6 Test Results"
+          files: src/test_report36.xml

Dockerfile.python3

@@ -7,6 +7,9 @@ ENV PYTHON_VERSION_MAJOR 3
 # Allow scripts to detect we're running in our own container
 RUN touch /addons-server-docker-container
 
+# Update the main repositories to the archived repository
+RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list
+
 # Add nodesource repository and requirements
 ADD docker/nodesource.gpg.key /etc/pki/gpg/GPG-KEY-nodesource
 RUN apt-get update && apt-get install -y \
@@ -17,7 +20,9 @@ RUN cat /etc/pki/gpg/GPG-KEY-nodesource | apt-key add -
 ADD docker/debian-stretch-nodesource-repo /etc/apt/sources.list.d/nodesource.list
 ADD docker/debian-stretch-backports-repo /etc/apt/sources.list.d/backports.list
 
-RUN apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y --allow-downgrades \
+        # Downgrades
+        libssl1.1=1.1.0l-1~deb9u1 \
         # General (dev-) dependencies
         bash-completion \
         build-essential \
@@ -50,16 +55,16 @@ RUN apt-get update && apt-get install -y \
         libmaxminddb-dev                 \
     && rm -rf /var/lib/apt/lists/*
 
-RUN apt-get update && apt-get -t stretch-backports install -y \
-        # For git-based files storage backend
-        libgit2-dev \
-    && rm -rf /var/lib/apt/lists/*
+#RUN apt-get update && apt-get -t stretch-backports install -y \
+#        # For git-based files storage backend
+#        libgit2-dev \
+#    && rm -rf /var/lib/apt/lists/*
 
-ADD http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz /tmp
+#ADD http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz /tmp
 
-RUN mkdir -p /usr/local/share/GeoIP \
- && gunzip -c /tmp/GeoLite2-Country.mmdb.gz > /usr/local/share/GeoIP/GeoLite2-Country.mmdb \
- && rm -f /tmp/GeoLite2-Country.mmdb.gz
+#RUN mkdir -p /usr/local/share/GeoIP \
+# && gunzip -c /tmp/GeoLite2-Country.mmdb.gz > /usr/local/share/GeoIP/GeoLite2-Country.mmdb \
+# && rm -f /tmp/GeoLite2-Country.mmdb.gz
 
 # Compile required locale
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
@@ -100,3 +105,4 @@ ENV CLEANCSS_BIN /deps/node_modules/.bin/cleancss
 ENV LESS_BIN /deps/node_modules/.bin/lessc
 ENV UGLIFY_BIN /deps/node_modules/.bin/uglifyjs
 ENV ADDONS_LINTER_BIN /deps/node_modules/.bin/addons-linter
+RUN npm cache clean -f && npm install -g n && /deps/bin/n 16

Makefile-docker

@@ -97,10 +97,10 @@ populate_data:
 	# reindex --wipe will force the ES mapping to be re-installed. Useful to
 	# make sure the mapping is correct before adding a bunch of add-ons.
 	$(PYTHON_COMMAND) manage.py reindex --wipe --force --noinput
-	$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
+	#$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
+	#$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
 	$(PYTHON_COMMAND) manage.py generate_addons --app thunderbird $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_themes --app firefox $(NUM_THEMES)
+	#$(PYTHON_COMMAND) manage.py generate_themes --app firefox $(NUM_THEMES)
 	$(PYTHON_COMMAND) manage.py generate_themes --app thunderbird $(NUM_THEMES)
 	# These add-ons are specifically useful for the addons-frontend
 	# homepage. You may have to re-run this, in case the data there

UPGRADE_STATUS.md

@@ -0,0 +1,3 @@
+# Upgrade Status
+
+ATN is currently rebased onto AMO tag 2019.02.28

docker-compose.mac-arm.yml

@@ -1,9 +1,7 @@
-version: "2.3"
-
 services:
-  nginx:
-    ports:
-      - "80:80"
+#  nginx:
+#    ports:
+#      - "80:80"
 
   web:
     ports:

docker-compose.override.yml

@@ -1,5 +1,3 @@
-version: "2.3"
-
 # If you're changing the &env mapping
 # please make sure to update tests/ui/docker-compose.selenium.yml
 # accordingly.
@@ -20,7 +18,10 @@ x-env-mapping: &env
 services:
   worker: &worker
     <<: *env
-    image: addons/addons-server:latest-py3
+    #image: addons/addons-server:latest-py3
+    build:
+      context: .
+      dockerfile: Dockerfile.python3
     command: supervisord -n -c /code/docker/supervisor-celery.conf
     entrypoint: ./scripts/start-docker.sh
     volumes:
@@ -32,19 +33,24 @@ services:
     <<: *worker
     command: supervisord -n -c /code/docker/supervisor.conf
 
-  nginx:
-    image: addons/addons-nginx
-    volumes:
-      - ./static:/srv/static
-      - ./site-static:/srv/site-static
-      - ./storage/shared_storage/uploads:/srv/user-media
-      - ./storage/files:/srv/user-media/addons
+#  nginx:
+#    image: addons/addons-nginx
+#    volumes:
+#      - ./static:/srv/static
+#      - ./site-static:/srv/site-static
+#      - ./storage/shared_storage/uploads:/srv/user-media
+#      - ./storage/files:/srv/user-media/addons
 
   memcached:
     image: memcached:1.4
 
   mysqld:
     image: mysql:5.7
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 20000
+        hard: 40000
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=yes
       - MYSQL_DATABASE=olympia
@@ -62,13 +68,18 @@ services:
       - xpack.watcher.enabled=false
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 20000
+        hard: 40000
     mem_limit: 2g
 
   redis:
     image: redis:2.8
 
   rabbitmq:
-    image: rabbitmq:3.5
+    image: rabbitmq:3
     hostname: olympia
     expose:
       - "5672"
@@ -77,19 +88,20 @@ services:
       - RABBITMQ_DEFAULT_PASS=olympia
       - RABBITMQ_DEFAULT_VHOST=olympia
 
+# ATN Doesn't sign addons, but we need it for tests
   autograph:
     image: mozilla/autograph:2.7.0
 
-  addons-frontend:
-    <<: *env
-    environment:
-      - HOSTNAME=uitests
-      - WEBPACK_SERVER_HOST=olympia-frontend.test
-      - FXA_CONFIG=default
-    image: addons/addons-frontend
-    ports:
-      - "3000:3000"
-      - "3001:3001"
-    command: yarn amo:ui-tests
-    links:
-      - "nginx:olympia.test"
+#  addons-frontend:
+#    <<: *env
+#    environment:
+#      - HOSTNAME=uitests
+#      - WEBPACK_SERVER_HOST=olympia-frontend.test
+#      - FXA_CONFIG=default
+#    image: addons/addons-frontend
+#    ports:
+#      - "3000:3000"
+#      - "3001:3001"
+#    command: yarn amo:ui-tests
+#    links:
+#      - "nginx:olympia.test"

docker-compose.yml

@@ -265,33 +265,8 @@ kombu==4.3.0 \
     --hash=sha256:7a2cbed551103db9a4e2efafe9b63222e012a61a18a881160ad797b9d4e1d0a1 \
     --hash=sha256:529df9e0ecc0bad9fc2b376c3ce4796c41b482cf697b78b71aea6ebe7ca353c8
 # lxml is required by pyquery
-lxml==4.3.1 \
-    --hash=sha256:1c630c083d782cbaf1f7f37f6cac87bda9cff643cf2803a5f180f30d97955cef \
-    --hash=sha256:983129f3fd3cef5c3cf067adcca56e30a169656c00fcc6c648629dbb850b27fa \
-    --hash=sha256:95b392952935947e0786a90b75cc33388549dcb19af716b525dae65b186138fc \
-    --hash=sha256:b5db89cc0ef624f3a81214b7961a99f443b8c91e88188376b6b322fd10d5b118 \
-    --hash=sha256:47d8365a8ef14097aa4c65730689be51851b4ade677285a3b2daa03b37893e26 \
-    --hash=sha256:9563a23c1456c0ab550c087833bc13fcc61013a66c6420921d5b70550ea312bf \
-    --hash=sha256:0537eee4902e8bf4f41bfee8133f7edf96533dd175930a12086d6a40d62376b2 \
-    --hash=sha256:7940d5c2185ffb989203dacbb28e6ae88b4f1bb25d04e17f94b0edd82232bcbd \
-    --hash=sha256:d4d63d85eacc6cb37b459b16061e1f100d154bee89dc8d8f9a6128a5a538e92e \
-    --hash=sha256:dccad2b3c583f036f43f80ac99ee212c2fa9a45151358d55f13004d095e683b2 \
-    --hash=sha256:c0c5a7d4aafcc30c9b6d8613a362567e32e5f5b708dc41bc3a81dac56f8af8bb \
-    --hash=sha256:7cf39bb3a905579836f7a8f3a45320d9eb22f16ab0c1e112efb940ced4d057a5 \
-    --hash=sha256:510e904079bc56ea784677348e151e1156040dbfb736f1d8ea4b9e6d0ab2d9f4 \
-    --hash=sha256:1ae4c0722fc70c0d4fba43ae33c2885f705e96dce1db41f75ae14a2d2749b428 \
-    --hash=sha256:09e91831e749fbf0f24608694e4573be0ef51430229450c39c83176cc2e2d353 \
-    --hash=sha256:a0b75b1f1854771844c647c464533def3e0a899dd094a85d1d4ed72ecaaee93d \
-    --hash=sha256:df46307d39f2aeaafa1d25309b8a8d11738b73e9861f72d4d0a092528f498baa \
-    --hash=sha256:0562ec748abd230ab87d73384e08fa784f9b9cee89e28696087d2d22c052cc27 \
-    --hash=sha256:ea825562b8cd057cbc9810d496b8b5dec37a1e2fc7b27bc7c1e72ce94462a09a \
-    --hash=sha256:2fe74e3836bd8c0fa7467ffae05545233c7f37de1eb765cacfda15ad20c6574a \
-    --hash=sha256:3f2d9eafbb0b24a33f56acd16f39fc935756524dcb3172892721c54713964c70 \
-    --hash=sha256:e70b5e1cb48828ddd2818f99b1662cb9226dc6f57d07fc75485405c77da17436 \
-    --hash=sha256:c0a7751ba1a4bfbe7831920d98cee3ce748007eab8dfda74593d44079568219a \
-    --hash=sha256:37af783c2667ead34a811037bda56a0b142ac8438f7ed29ae93f82ddb812fbd6 \
-    --hash=sha256:58d0851da422bba31c7f652a7e9335313cf94a641aa6d73b8f3c67602f75b593 \
-    --hash=sha256:da5e7e941d6e71c9c9a717c93725cda0708c2474f532e3680ac5e39ec57d224d
+lxml==4.3.2 \
+    --hash=sha256:4888be27d5cba55ce94209baef5bcd7bbd7314a3d17021a5fc10000b3a5f737d
 # m2secret is required by django-aesfield
 m2secret-py3==1.3 \
     --hash=sha256:94dfae937e8a61bf48812fab6a0b8de300781c4e1dbee0cee3aa57cb01915864