Merge pull request #637 from RNAcentral/update-gunicorn

Update gunicorn

Author: caduribas

.github/workflows/main.yml

@@ -10,7 +10,6 @@ name: Building Docker containers
 on:
   push:
     paths-ignore:
-      - 'jenkins/**'
       - 'kubernetes/**'
       - 'nginx/**'
       - 'openssl/**'

Dockerfile

@@ -70,4 +70,4 @@ USER rnacentral
 COPY ./entrypoint.sh $RNACENTRAL_HOME
 ENTRYPOINT ["/srv/rnacentral/rnacentral-webcode/entrypoint.sh"]
 
-CMD ["gunicorn", "--chdir", "/srv/rnacentral/rnacentral-webcode/rnacentral", "--bind", "0.0.0.0:8000", "rnacentral.wsgi:application", "--workers", "2", "--timeout", "120", "--max-requests", "1000", "--max-requests-jitter", "100", "--log-level=debug", "--access-logfile", "/dev/stdout", "--error-logfile", "/dev/stderr"]
+CMD ["gunicorn", "--chdir", "/srv/rnacentral/rnacentral-webcode/rnacentral", "--bind", "0.0.0.0:8000", "rnacentral.wsgi:application", "--workers", "4", "--threads", "2", "--timeout", "120", "--graceful-timeout", "60",  "--keep-alive", "10", "--max-requests", "1000", "--max-requests-jitter", "100", "--log-level=debug", "--access-logfile", "/dev/stdout", "--error-logfile", "/dev/stderr"]

entrypoint.sh

@@ -57,6 +57,7 @@ else
             "PASSWORD": "$DB_PASSWORD",
             "HOST": "$DB_HOST",
             "PORT": "$DB_PORT",
+            "CONN_MAX_AGE": 0
         }
     }
 	EOF

kubernetes/helm/templates/nginx.yaml

@@ -30,6 +30,10 @@ data:
         access_log stdout;
         error_log  stderr;
 
+        if ($http_user_agent ~* (Ahrefs|MJ12bot|Semrush|Yandex|facebookexternal|developers\.facebook\.com)) {
+            return 403;
+        }
+
         location /static/ {
             autoindex on;
             alias /srv/rnacentral/static/;
@@ -41,6 +45,25 @@ data:
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header Host $host;
             proxy_redirect off;
+
+            proxy_read_timeout 120;
+            proxy_send_timeout 120;
+            proxy_connect_timeout 120;
+        }
+
+        error_page 500 502 503 504 /error/;
+        location /error/ {
+            internal;
+            proxy_pass http://rnacentral/error/;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header Host $host;
+            proxy_redirect off;
+        }
+
+        location /nginx_status {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
         }
     }
 ---

locust/locustfile.py

@@ -0,0 +1,29 @@
+######
+# This tool was used to simulate access to the website
+# by hundreds of users and check the behavior of the
+# server and database. Run the load testing tool with:
+#
+# locust -f locustfile.py --host=http://localhost:8010
+#
+######
+
+import random
+
+from locust import HttpUser, TaskSet, between, task
+
+# The urs_list.txt file is just an example and therefore
+# contains a small list of URS
+with open("urs_list.txt", "r") as f:
+    urs = [line.strip() for line in f.readlines()]
+
+
+class UserBehavior(TaskSet):
+    @task
+    def fetch_sequence_page(self):
+        random_urs = random.choice(urs)
+        self.client.get(f"/rna/{random_urs}")
+
+
+class WebsiteUser(HttpUser):
+    tasks = [UserBehavior]
+    wait_time = between(1, 3)

locust/urs_list.txt

@@ -0,0 +1,10 @@
+URS0000000055_9606
+URS0000000096_9606
+URS0000000098_9606
+URS00000000C9_9606
+URS00000000D2_9606
+URS00000000FD_9606
+URS0000000185_9606
+URS000000022A_9606
+URS000000023A_9606
+URS00000002D9_9606

nginx/conf.d/local.conf

@@ -5,6 +5,14 @@ upstream rnacentral {
 
 server {
     listen 80;
+    root /srv/rnacentral/static/;
+    access_log stdout;
+    error_log  stderr;
+
+    location /static/ {
+        autoindex on;
+        alias /srv/rnacentral/static/;
+    }
 
     location / {
         # everything is passed to Gunicorn
@@ -15,11 +23,15 @@ server {
 
         proxy_read_timeout 120;
         proxy_send_timeout 120;
-        proxy_connect_timeout 75;
+        proxy_connect_timeout 120;
     }
 
-    location /static/ {
-        autoindex on;
-        alias /srv/rnacentral/static/;
+    error_page 500 502 503 504 /error/;
+    location /error/ {
+        internal;
+        proxy_pass http://rnacentral/error/;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $host;
+        proxy_redirect off;
     }
 }

rnacentral/portal/templates/portal/base.html

@@ -89,6 +89,16 @@
         <link rel="apple-touch-icon" sizes="144x144" href="{% static "img/icons/apple-touch-icon-144x144.png" %}" />
         <link rel="apple-touch-icon" sizes="152x152" href="{% static "img/icons/apple-touch-icon-152x152.png" %}" />
 
+        <!-- Google tag (gtag.js) -->
+        <script async src="https://www.googletagmanager.com/gtag/js?id=G-37T22PHH90"></script>
+        <script>
+          window.dataLayer = window.dataLayer || [];
+          function gtag(){dataLayer.push(arguments);}
+          gtag('js', new Date());
+
+          gtag('config', 'G-37T22PHH90');
+        </script>
+
         {% block extra_head %}{% endblock %}
     </head>
 
@@ -260,16 +270,6 @@
         {% block extra_js_uncompressed %}
         {% endblock %}
 
-        <!-- Google tag (gtag.js) -->
-        <script async src="https://www.googletagmanager.com/gtag/js?id=G-37T22PHH90"></script>
-        <script>
-          window.dataLayer = window.dataLayer || [];
-          function gtag(){dataLayer.push(arguments);}
-          gtag('js', new Date());
-
-          gtag('config', 'G-37T22PHH90');
-        </script>
-
         <!--Doorbell.io feedback-->
         <script type="text/javascript">
             window.doorbellOptions = {

rnacentral/requirements_dev.txt

@@ -7,3 +7,6 @@ mock==4.0.3
 
 # pre-commit hooks
 pre-commit==2.20.0
+
+# performance/load testing tool
+locust==2.32.3

rnacentral/templates/robots.txt

@@ -1,14 +1,20 @@
 User-agent: AhrefsBot
+User-agent: CCBot
+User-agent: ClaudeBot
 User-agent: DotBot
+User-agent: FacebookBot
+User-agent: GPTBot
 User-agent: MJ12bot
 User-agent: PetalBot
+User-agent: SeekportBot
 User-agent: SEMrushBot
 Disallow: /
-Disallow: /sitemap.xml
 
 User-agent: *
+Crawl-delay: 5
+Disallow: /api/current/
+Disallow: /api/v1/
 Disallow: /export/
 Disallow: /sequence-search/
-Allow: /sitemap.xml
 
 Sitemap: https://rnacentral.org/sitemap.xml