zodbconvert example config and scripts

Author: fredvd

devops/zodbconvert/README.txt

@@ -1,23 +1,22 @@
 Convert postgresql to filestorage
 =================================
 
-In test and production the plone.org website stores the content data in a
+In test and production the website stores the content data in a
 Postgresql database using the relstorage driver. If you want to create
 a local filestorage copy of the data to use for local development, you
 can use the zodbconvert utility.
 
-First make sure you are running a local postgresql server with the Plone
-content database. The zodbconvert config file assume you use the (datbase)
-settings as configured in the docker-compose.yml in the project root. This
-compose file starts a local postgresql container on 127.0.0.1:5432 with
-ploneorg/ploneorg/ploneorg as the database/role/password . 
+First make sure you are running a local postgresql server with the site
+content database. The zodbconvert config file assume you use the (database)
+settings as configured in the postgres-compose.yml in the project root. This
+compose file starts a local postgresql container on 127.0.0.1:5432 
 
 zodbconvert writes the data to the ./data directory in the project root,
 inside ./data/filestorage and ./data/blobs .  
 
 Now run zodbconvert from the project root: 
 
-> ./backend/bin/zodbconvert ./devops/zodbconvert/relstorage.cfg
+> ./backend/bin/zodbconvert ./devops/zodbconvert/from-relstorage.cfg
 
-This can take 5-15 minutes, depending on your machine specs. 
+This can take some minutes, depending on your machine specs. 
 

devops/zodbconvert/docker-compose.yml

@@ -0,0 +1,15 @@
+version: "3.8"
+
+services:
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: zodbconvert
+      POSTGRES_PASSWORD: zodbconvert
+      POSTGRES_DB: zodbconvert
+    command: postgres -c shared_buffers=2GB -c effective_cache_size=4GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - ./data/postgresql:/var/lib/postgresql/data

devops/zodbconvert/export_filestorage.sh

@@ -0,0 +1 @@
+/bin/zodbconvert -f ./from-relstorage.cfg

devops/zodbconvert/export_filestorage_tarball.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# import pgdump file:
+# docker exec -i zodbconvert-db-1 pg_restore -U zodbconvert -d zodbconvert --clean --no-owner -v < $1
+
+rm -rf ./data/filestorage ./data-blobstorage
+mkdir -p ./data/filestorage ./data/blobstorage
+./bin/zodbconvert ./from-relstorage.cfg
+cd data
+tar cfz ploneorg_devdata.tar.gz  filestorage blobstorage
+mv ploneorg_devdatatar.gz ..
+cd ..

devops/zodbconvert/relstorage.cfg devops/zodbconvert/from-relstorage.cfgdevops/zodbconvert/relstorage.cfg renamed to devops/zodbconvert/from-relstorage.cfg

@@ -2,10 +2,10 @@
   blob-dir /tmp/blobcache
   shared-blob-dir false 
   <postgresql>
-    dsn dbname='ploneorg' user='ploneorg' host='127.0.0.1' port='5432' password='ploneorg'
+    dsn dbname='zodbconvert' user='zodbconvert' host='127.0.0.1' port='5432' password='zodbconvert'
   </postgresql>
 </relstorage>
 <filestorage destination>
   path ./data/filestorage/Data.fs
-  blob-dir ./data/blobs
+  blob-dir ./data/blobstorage
 </filestorage>

devops/zodbconvert/requirements.txt

@@ -0,0 +1,2 @@
+relstorage==4.0.0
+psycopg2==2.9.9

devops/zodbconvert/restore_dump_to_postgresql.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker exec -i zodbconvert-db-1 pg_restore -U zodbconvert -d zodbconvert --clean --no-owner -v < $1

devops/zodbconvert/to-relstorage.cfg

@@ -0,0 +1,11 @@
+<filestorage source>
+  path ../../data/filestorage/Data.fs
+  blob-dir ../../data/blobs
+</filestorage>
+<relstorage destination>
+  blob-dir /tmp/blobcache
+  shared-blob-dir false 
+  <postgresql>
+    dsn dbname='zodbconvert' user='zodbconvert' host='127.0.0.1' port='5432' password='zodbconvert'
+  </postgresql>
+</relstorage>