Clean up old -fs targets in Makefile.

Use instance-local.yaml instead to override for example the data storage layer of the backend from relstoraeg to file/blobstorage.

Author: fredvd

backend/Makefile

@@ -94,11 +94,12 @@ bin/pip:
 config: bin/pip  ## Create instance configuration
 	@echo "$(GREEN)==> Create instance configuration$(RESET)"
 	bin/cookiecutter -f --no-input --config-file instance.yaml --checkout 2.1.0 gh:plone/cookiecutter-zope-instance
-
-.PHONY: config-fs
-config-fs: bin/pip  ## Create instance configuration
-	@echo "$(GREEN)==> Create instance configuration$(RESET)"
-	bin/cookiecutter -f --no-input --config-file instance-filestorage.yaml --checkout 2.1.0 gh:plone/cookiecutter-zope-instance
+	if [[ -s instance-local.yaml ]]; then \
+				echo ">>> Using instance-local.yaml"; \
+				bin/cookiecutter -f --no-input --config-file instance-local.yaml  --checkout 2.1.0 gh:plone/cookiecutter-zope-instance ;\
+	else \
+				bin/cookiecutter -f --no-input --config-file instance.yaml  --checkout 2.1.0  gh:plone/cookiecutter-zope-instance ;\
+	fi
 
 # i18n
 bin/i18ndude: bin/pip
@@ -120,13 +121,6 @@ build-dev: config ## pip install Plone packages
 	@bin/mxdev -c mx.ini
 	bin/pip install -r requirements-mxdev.txt
 
-.PHONY: build-dev-fs
-build-dev-fs: config-fs ## pip install Plone packages
-	@echo "$(GREEN)==> Setup Build$(RESET)"
-	sed "s/PLONE_VERSION/$(PLONE_VERSION)/g" constraints-template.txt > constraints.txt
-	@bin/mxdev -c mx.ini
-	bin/pip install -r requirements-mxdev.txt
-
 .PHONY: format
 format: ## Format the codebase according to our standards
 	@echo "$(GREEN)==> Format codebase$(RESET)"

backend/instance-filestorage.yaml renamed to backend/instance-local.yaml.example

@@ -1,3 +1,8 @@
+# If you rename this file to instance-local.yaml, the Makefile config/build commands
+# will use this file for local development
+# In it you can override the backend setup to use for example file/blobstorage
+# instead of the relstorage connection used in testing/production deployment
+
 default_context:
     initial_user_name: 'admin'
     initial_user_password: 'adminadmin'

backend/scripts/catalog_clear_rebuild.py

@@ -0,0 +1,60 @@
+# Run this from the backend directory with:
+# bin/zconsole run instance/etc/zope.conf scripts/catalog_clear_rebuild.py
+# or with extra options: --site=nl
+
+from plone import api
+from zope.component import getUtility
+from zope.component.hooks import setSite
+
+import argparse
+import sys
+import transaction
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--site",
+    default="",
+    dest="site",
+    help="Single site id to work on. Default is to work on all.",
+)
+# sys.argv will be something like:
+# ['.../parts/instance/bin/interpreter', '-c',
+#  'scripts/fix_related_items_and_intids.py', '--site=nl']
+# Ignore the first three.
+options = parser.parse_args(args=sys.argv[3:])
+
+# 'app' is the Zope root.
+# Get Plone Sites to work on.
+if options.site:
+    # Get single Plone Site.
+    plones = [getattr(app, options.site)]
+else:
+    # Get all Plone Sites.
+    plones = [
+        obj
+        for obj in app.objectValues()  # noqa
+        if getattr(obj, "portal_type", "") == "Plone Site"
+    ]
+
+
+def commit(note):
+    print(note)
+    # Commit transaction and add note.
+    tr = transaction.get()
+    tr.note(note)
+    transaction.commit()
+
+
+for site in plones:
+    print("")
+    print("Handling Plone Site %s." % site.id)
+    setSite(site)
+    catalog = api.portal.get_tool(name="portal_catalog")
+    catalog.manage_catalogRebuild()
+    print("Catalog is rebuilt, now doing a search to empty the indexing queue. This may take long.")
+    catalog.unrestrictedSearchResults(SearchableText="Maro 1:3")
+    note = (
+        "Cleared and rebuilt the catalog for %s." % site.id
+    )
+    commit(note)
+    print("Done.")

backend/scripts/purge_image_scales.py

@@ -0,0 +1,88 @@
+# Run this with:
+# bin/zconsole run instance/etc/zope.conf scripts/purge_image_scales.py
+#
+# Add --dry-run to change nothing and only get a report.
+
+import sys
+import transaction
+from Products.CMFCore.utils import getToolByName
+from plone.scale.storage import AnnotationStorage
+from zope.component.hooks import setSite
+
+# Keep scales of at most X days older than their context:
+DAYS = -1
+# Commit after these many changes:
+LIMIT = 1000
+
+if '--dry-run' in sys.argv:
+    dry_run = True
+    print('Dry run selected, will not commit changes.')
+else:
+    dry_run = False
+
+# Get all Plone Sites.  'app' is the Zope root.
+plones = [obj for obj in app.objectValues()
+          if getattr(obj, 'portal_type', '') == 'Plone Site']
+
+
+def commit(note):
+    print(note)
+    if dry_run:
+        print('Dry run selected, not committing.')
+        return
+    # Commit transaction and add note.
+    tr = transaction.get()
+    tr.note(note)
+    transaction.commit()
+
+
+for site in plones:
+    print('')
+    print('Handling Plone Site %s.' % site.id)
+    setSite(site)
+    catalog = getToolByName(site, 'portal_catalog')
+    count = 0
+    purged = 0
+    for brain in catalog.getAllBrains():
+        try:
+            obj = brain.getObject()
+        except:
+            continue
+        savepoint = transaction.savepoint()
+        ann = AnnotationStorage(obj)
+        try:
+            ann.storage
+        except TypeError:
+            # This happens when the context cannot be annotated, for
+            # example for a plone.app.discussion comment.
+            continue
+        # We want to remove all scales that are X days older than the
+        # last modification date of the object.
+        final_date = obj.modified() - DAYS
+        changed = False
+        to_delete = []
+        for key, value in ann.items():
+            if value['modified'] < final_date.millis():
+                to_delete.append(key)
+        for key in to_delete:
+            # This may easily give an error, as it tries to remove
+            # two keys: del ann[key]
+            del ann.storage[key]
+            purged += 1
+            changed = True
+        if not changed:
+            # This avoids adding an empty annotation for items that
+            # will never store scales.
+            savepoint.rollback()
+        else:
+            count += 1
+            if count % LIMIT == 0:
+                note = ('Purged %d outdated image scales for %d items in '
+                        'Plone Site %s.' % (purged, count, site.id))
+                commit(note)
+
+    note = ('Finished purging %d outdated image scales for %d items in '
+            'Plone Site %s.' % (purged, count, site.id))
+    commit(note)
+
+print('Done.')