Merge pull request #1504 from samvera/fedora6_rebase

Rebase Fedora 6 work on ActiveFedora 15

Author: cjcolvar

.circleci/config.yml

@@ -4,6 +4,9 @@ orbs:
 jobs:
   bundle_lint_test:
     parameters:
+      ruby_type:
+        type: string
+        default: 'ruby'
       ruby_version:
         type: string
       bundler_version:
@@ -14,23 +17,22 @@ jobs:
       solr_config_path:
         type: string
         default: lib/generators/active_fedora/config/solr/templates/solr/conf
-      ruby_type:
-        type: string
-        default: 'ruby'
       solr_port:
         type: string
         default: '8985'
       fcrepo_version:
         type: string
-        default: "4.7.5"
+        default: "5.1.0"
       solr_version:
         type: string
         default: "9"
     docker:
       - image: cimg/<< parameters.ruby_type >>:<< parameters.ruby_version >>-browsers
-      - image: samvera/fcrepo4:<< parameters.fcrepo_version >>
+      - image: fcrepo/fcrepo:6.5-tomcat9
         environment:
-          CATALINA_OPTS: "-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC"
+          JAVA_OPTS: -Dfcrepo.autoversioning.enabled=false -Dfcrepo.external.content.allowed=/fcrepo_allow.txt
+        command: |
+          /bin/bash -c "echo 'http://fedoraAdmin:fedoraAdmin@localhost:8080/' > /fcrepo_allow.txt && catalina.sh run"
       - image: zookeeper:3.9
         environment:
           ZOO_ADMINSERVER_ENABLED: false
@@ -49,6 +51,8 @@ jobs:
       FCREPO_TEST_PORT: 8080
       SPEC_OPTS: --profile 10 --format RspecJunitFormatter --out /tmp/test-results/rspec.xml --format progress
       RAILS_VERSION: << parameters.rails_version >>
+      FCREPO_PORT: 8080
+      FCREPO_REST_PATH: fcrepo/rest
     steps:
       - samvera/cached_checkout
       - checkout

.fcrepo_wrapper

@@ -0,0 +1,10 @@
+# Place any default configuration for solr_wrapper here
+port: 8986
+version: 6.4.0
+enable_jms: false
+java_options: ['-Dfcrepo.log.http.api=DEBUG','-Dfcrepo.log.kernel=ERROR','-Xmx512m',
+  '-Dfcrepo.external.content.allowed=config/fcrepo_allow.txt', '-Dfcrepo.autoversioning.enabled=false']
+#managed: true
+#fcrepo_home_dir: tmp/fcrepo-development-data
+url: https://github.com/fcrepo/fcrepo/releases/download/fcrepo-6.4.0/fcrepo-webapp-6.4.0-jetty-console.jar
+ignore_md5sum: true

.rubocop.yml

@@ -248,6 +248,9 @@ Rails/FilePath:
 Rails/ActiveRecordAliases:
   Enabled: false
 
+Rails/RakeEnvironment:
+  Enabled: false
+
 Rails/SkipsModelValidations:
   Enabled: false
 

.solr_wrapper

@@ -1,5 +1,5 @@
 # Place any default configuration for solr_wrapper here
-# version: 6.1.0
+version: 7.7.3
 port: 8985
 collection:
   dir: lib/generators/active_fedora/config/solr/templates/solr/conf/

Upgrading.md

@@ -0,0 +1,46 @@
+# Upgrading to ActiveFedora 16+
+
+ActiveFedora 16+ is built for use with Fedora 6+.
+
+## Changes from ActiveFedora 15
+
+- Direct/Indirect container classes now specified in headers required by Fedora 5+
+- External files supported via a Link header (see ActiveFedora::File::External)
+- Versioning updated based upon Fedora's use of Memento
+- Ability to restore previous versions removed
+
+## Upgrade considerations
+
+- You may want to turn off autoversioning in Fedora (`fcrepo.autoversioning.enabled=false`) if you edit objects frequently.
+- Be sure to allocate sufficient memory (2-4 GB) to the import-export tools.
+- The Fedora 4 export and Fedora 5/6 data directories have lots of files so file system operations can take a LONG time.
+
+## Upgrading from Fedora 4 (without versions)
+
+1. Install Java 21+
+2. Download [fcrepo-import-export](https://github.com/fcrepo-exts/fcrepo-import-export/) 1.2.0 or higher.
+3. Export from Fedora 4 (update --resource url to match the root of your fedora instance):
+```
+java -jar fcrepo-import-export-1.2.0.jar -b --dir fcrepo4.7.5_export --user fedoraAdmin:fedoraAdmin --mode export --resource http://fedora:8080/fedora/rest --binaries --membership --auditLog > importexport_`date +%Y%m%dT%H%M%S`.log 2>&1
+```
+4. Check that all objects were exported (uses ripgrep):
+```
+rg '^<(http://fedora:8080/fedora/[^#:]*)>$' -g '*.ttl' fcrepo4.7.5_export/ --no-heading -m 1 -INor '$1' | sort | uniq > subject_ids
+rg '<(http://fedora:8080/fedora/[^#:]*)>.*[.;]$' -g '*.ttl' fcrepo4.7.5_export/ --no-heading -INor '$1' | sort | uniq > object_ids
+comm -13 subject_ids object_ids > missing_ids
+```
+5. (If necessary) Restart export from missing_ids file (can also be done with remaining file if export does not finish successfully):
+```
+java -jar fcrepo-import-export-1.2.0.jar --dir fcrepo4.7.5_export --user fedoraAdmin:fedoraAdmin --mode export --repositoryRoot http://fedora:8080/fedora/rest --resourcesFile missing_ids --binaries --membership --auditLog > importexport_`date +%Y%m%dT%H%M%S`.log 2>&1
+``` 
+6. Download [fcrepo-upgrade-utils](https://github.com/fcrepo-exts/fcrepo-upgrade-utils/) 6.4.0 or higher.  (6.4.0 has not been released yet so use the [Avalon patched version](https://github.com/avalonmediasystem/fcrepo-upgrade-utils/releases/download/6.3.0-AVALON/fcrepo-upgrade-utils-6.3.0-AVALON.jar) for now.)
+7. Migrate Fedora 4 export to Fedora 5:
+```
+java -jar fcrepo-upgrade-utils-6.3.0-AVALON.jar --input-dir fcrepo4.7.5_export --output-dir fcrepo5_export --source-version 4.7.5 --target-version 5+ > upgrade_5_`date +%Y%m%dT%H%M%S`.log 2>&1
+```
+8. Migrate Fedora 5 data directory to Fedora 6 (be sure that --base-uri matches --resource from the Fedora 4 export):
+```
+java --add-opens java.base/java.util.concurrent=ALL-UNNAMED -jar fcrepo-upgrade-utils-6.3.0-AVALON.jar --input-dir fcrepo5_export --output-dir fcrepo6_export  --source-version 5+ --target-version 6+ --base-uri http://fedora:8080/fedora/rest > upgrade_6_`date +%Y%m%dT%H%M%S`.log 2>&1
+```
+9. Copy Fedora 6 data directory to `fcrepo.home` and startup Fedora
+This will kick off Fedora's indexing which can take hours depending on the size of your repository.  During indexing Fedora will not respond to requests.

active-fedora.gemspec

@@ -20,8 +20,9 @@ Gem::Specification.new do |s|
   s.add_dependency "activesupport", '>= 6.1'
   s.add_dependency "active-triples", '>= 0.11.0', '< 2.0.0'
   s.add_dependency "deprecation"
-  s.add_dependency 'faraday', '>= 1.0'
+  s.add_dependency 'faraday', '>= 2.0'
   s.add_dependency 'faraday-encoding', '>= 0.0.5'
+  s.add_dependency 'faraday-follow_redirects'
   s.add_dependency "ldp", '>= 0.7.0', '< 2'
   s.add_dependency "mutex_m"
   s.add_dependency 'rsolr', '>= 1.1.2', '< 3'
@@ -47,4 +48,5 @@ Gem::Specification.new do |s|
     "README.md"
   ]
   s.require_paths = ["lib"]
+  s.post_install_message = "ActiveFedora 16+ is built for Fedora 6+.  See Upgrading.md for important changes from ActiveFedora 15 and hints for using the Fedora upgrade tools to migrate from Fedora 4."
 end

config/fcrepo_allow.txt

@@ -0,0 +1,3 @@
+https://example.com/
+http://example.com/
+http://localhost:8986/

lib/active_fedora/attached_files.rb

@@ -80,6 +80,8 @@ def add_file(file, opts)
         node.content = file
         node.mime_type = opts[:mime_type]
         node.original_name = opts[:original_name]
+        node.external_uri = opts[:external_uri]
+        node.external_handling = opts[:external_handling]
       end
     end
 

lib/active_fedora/change_set.rb

@@ -32,7 +32,9 @@ def changes
         elsif key == 'type'.freeze
           # working around https://github.com/ActiveTriples/ActiveTriples/issues/122
           predicate = ::RDF.type
-          result[predicate] = graph.query({ subject: object.rdf_subject, predicate: predicate })
+          result[predicate] = graph.query({ subject: object.rdf_subject, predicate: predicate }).select do |statement|
+            !statement.object.to_s.start_with?("http://fedora.info/definitions/v4/repository#", "http://www.w3.org/ns/ldp#")
+          end
         elsif object.local_attributes.include?(key)
           raise "Unable to find a graph predicate corresponding to the attribute: \"#{key}\""
         end

lib/active_fedora/clean_connection.rb

@@ -13,6 +13,7 @@ def get(*args)
 
       def omit_uris
         [
+          "http://fedora.info/definitions/fcrepo#ServerManaged",
           ::RDF::Vocab::Fcrepo4.ServerManaged,
           ::RDF::Vocab::LDP.PreferContainment,
           ::RDF::Vocab::LDP.PreferEmptyContainer,