Merge pull request #12394 from uncch-rdmc/12036-remove-minio-as-s3-storage-option

Replace MinIO with LocalStack as an S3 storage option

Author: pdurbin

conf/keycloak/docker-compose-dev.yml

@@ -53,16 +53,16 @@ services:
         -Ddataverse.files.localstack1.download-redirect=true
         -Ddataverse.files.localstack1.access-key=default
         -Ddataverse.files.localstack1.secret-key=default
-        -Ddataverse.files.minio1.type=s3
-        -Ddataverse.files.minio1.label=MinIO
-        -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000
-        -Ddataverse.files.minio1.custom-endpoint-region=us-east-1
-        -Ddataverse.files.minio1.bucket-name=mybucket
-        -Ddataverse.files.minio1.path-style-access=true
-        -Ddataverse.files.minio1.upload-redirect=false
-        -Ddataverse.files.minio1.download-redirect=false
-        -Ddataverse.files.minio1.access-key=4cc355_k3y
-        -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y
+        -Ddataverse.files.localstack_noredirect.type=s3
+        -Ddataverse.files.localstack_noredirect.label=LocalStackNoRedirect
+        -Ddataverse.files.localstack_noredirect.custom-endpoint-url=http://localstack:4566
+        -Ddataverse.files.localstack_noredirect.custom-endpoint-region=us-east-2
+        -Ddataverse.files.localstack_noredirect.bucket-name=mybucket-noredirect
+        -Ddataverse.files.localstack_noredirect.path-style-access=true
+        -Ddataverse.files.localstack_noredirect.upload-redirect=false
+        -Ddataverse.files.localstack_noredirect.download-redirect=false
+        -Ddataverse.files.localstack_noredirect.access-key=default
+        -Ddataverse.files.localstack_noredirect.secret-key=default
         -Ddataverse.pid.providers=fake
         -Ddataverse.pid.default-provider=fake
         -Ddataverse.pid.fake.type=FAKE
@@ -260,23 +260,6 @@ services:
     tmpfs:
       - /localstack:mode=770,size=128M,uid=1000,gid=1000
 
-  dev_minio:
-    container_name: "dev_minio"
-    hostname: "minio"
-    image: minio/minio
-    restart: on-failure
-    ports:
-      - "9000:9000"
-      - "9001:9001"
-    networks:
-      - dataverse
-    volumes:
-      - ./docker-dev-volumes/minio_storage:/data
-    environment:
-      MINIO_ROOT_USER: 4cc355_k3y
-      MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k3y
-    command: server /data
-
   previewers-provider:
     container_name: previewers-provider
     hostname: previewers-provider

conf/localstack/buckets.sh

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 # https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack
 awslocal s3 mb s3://mybucket
+awslocal s3 mb s3://mybucket-noredirect

doc/sphinx-guides/source/installation/big-data-support.rst

@@ -68,7 +68,6 @@ If the bucket allows the wildcard ``*`` but the Dataverse application only allow
 Detailed information for the most common S3 admin tools around CORS:
 
 - `AWS <https://docs.aws.amazon.com/AmazonS3/latest/userguide/enabling-cors-examples.html>`_
-- `Minio mc <https://docs.min.io/enterprise/aistor-object-store/reference/cli/mc-cors>`_
 - `s3cmd <https://servicedesk.surf.nl/wiki/spaces/WIKI/pages/215253125/Object+Store+S3+CORS+Policies>`_
 
 Get Current CORS Policy on Bucket
@@ -80,9 +79,6 @@ If you'd like to check the CORS configuration on your bucket before making chang
    .. group-tab:: AWS CLI
       :code:`aws s3api get-bucket-cors --bucket <BUCKET_NAME>`
 
-   .. group-tab:: Minio Client (mc)
-      :code:`mc cors get <STORE_NAME>/<BUCKET_NAME>`
-
 Set CORS Policy on Bucket
 +++++++++++++++++++++++++
 
@@ -107,9 +103,6 @@ Both JSON and XML format are explained in detail in `AWS Docs <https://docs.aws.
 
     Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above.
 
-  .. group-tab:: Minio Client (mc)
-    Create a file :download:`cors.xml </_static/installation/cors/cors.xml>` as follows:
-
     .. literalinclude:: /_static/installation/cors/cors.xml
         :name: xml-cors
         :language: xml
@@ -124,7 +117,7 @@ Both JSON and XML format are explained in detail in `AWS Docs <https://docs.aws.
 S3 Tags and Direct Upload
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide.
+Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide.
 
 Trusted Remote Storage with the ``remote`` Store Type
 -----------------------------------------------------

doc/sphinx-guides/source/installation/config.rst

@@ -1218,7 +1218,7 @@ You can configure this redirect properly in your cloud environment to generate a
 Amazon S3 Storage (or Compatible)
 +++++++++++++++++++++++++++++++++
 
-The Dataverse Software supports Amazon S3 storage as well as other S3-compatible stores (like Minio, Ceph RADOS S3 Gateway and many more) for files uploaded to your Dataverse installation.
+The Dataverse Software supports Amazon S3 storage as well as other S3-compatible stores (like Ceph RADOS S3 Gateway and many more) for files uploaded to your Dataverse installation.
 
 The Dataverse Software S3 driver supports multi-part upload for large files (over 1 GB by default - see the min-part-size option in the table below to change this).
 
@@ -1264,7 +1264,7 @@ Please make note of the following details:
 
 - **Endpoint URL** - consult the documentation of your service on how to find it.
 
-  * Example: https://play.minio.io:9000
+  * Example: http://localhost.localstack.cloud:4566
 
 - **Region:** Optional, but some services might use it. Consult your service documentation.
 
@@ -1461,11 +1461,6 @@ You may provide the values for these via any `supported MicroProfile Config API
 Reported Working S3-Compatible Storage
 ######################################
 
-`Minio v2018-09-12 <https://minio.io>`_
-  Set ``dataverse.files.<id>.path-style-access=true``, as Minio works path-based. Works pretty smooth, easy to setup.
-  **Can be used for quick testing, too:** just use the example values above. Uses the public (read: unsecure and
-  possibly slow) https://play.minio.io:9000 service.
-
 `StorJ Object Store <https://www.storj.io>`_
  StorJ is a distributed object store that can be configured with an S3 gateway. Per the S3 Storage instructions above, you'll first set up the StorJ S3 store by defining the id, type, and label. After following the general installation, set the following configuration to use a StorJ object store: ``dataverse.files.<id>.chunked-encoding=false``. For step-by-step instructions see https://docs.storj.io/dcs/how-tos/dataverse-integration-guide/
 

docker-compose-dev.yml

@@ -44,16 +44,16 @@ services:
         -Ddataverse.files.localstack1.download-redirect=true
         -Ddataverse.files.localstack1.access-key=default
         -Ddataverse.files.localstack1.secret-key=default
-        -Ddataverse.files.minio1.type=s3
-        -Ddataverse.files.minio1.label=MinIO
-        -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000
-        -Ddataverse.files.minio1.custom-endpoint-region=us-east-1
-        -Ddataverse.files.minio1.bucket-name=mybucket
-        -Ddataverse.files.minio1.path-style-access=true
-        -Ddataverse.files.minio1.upload-redirect=false
-        -Ddataverse.files.minio1.download-redirect=false
-        -Ddataverse.files.minio1.access-key=4cc355_k3y
-        -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y
+        -Ddataverse.files.localstack_noredirect.type=s3
+        -Ddataverse.files.localstack_noredirect.label=LocalStackNoRedirect
+        -Ddataverse.files.localstack_noredirect.custom-endpoint-url=http://localstack:4566
+        -Ddataverse.files.localstack_noredirect.custom-endpoint-region=us-east-2
+        -Ddataverse.files.localstack_noredirect.bucket-name=mybucket-noredirect
+        -Ddataverse.files.localstack_noredirect.path-style-access=true
+        -Ddataverse.files.localstack_noredirect.upload-redirect=false
+        -Ddataverse.files.localstack_noredirect.download-redirect=false
+        -Ddataverse.files.localstack_noredirect.access-key=default
+        -Ddataverse.files.localstack_noredirect.secret-key=default
         -Ddataverse.pid.providers=fake
         -Ddataverse.pid.default-provider=fake
         -Ddataverse.pid.fake.type=FAKE
@@ -252,23 +252,6 @@ services:
     tmpfs:
       - /localstack:mode=770,size=128M,uid=1000,gid=1000
 
-  dev_minio:
-    container_name: "dev_minio"
-    hostname: "minio"
-    image: minio/minio
-    restart: on-failure
-    ports:
-      - "9000:9000"
-      - "9001:9001"
-    networks:
-      - dataverse
-    volumes:
-      - ./docker-dev-volumes/minio_storage:/data
-    environment:
-      MINIO_ROOT_USER: 4cc355_k3y
-      MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k3y
-    command: server /data
-
   previewers-provider:
     container_name: previewers-provider
     hostname: previewers-provider

scripts/dev/dev-start-frd.sh

@@ -28,7 +28,6 @@ mkdir -p docker-dev-volumes/app/secrets
 mkdir -p docker-dev-volumes/postgresql/data
 mkdir -p docker-dev-volumes/solr/data
 mkdir -p docker-dev-volumes/solr/conf
-mkdir -p docker-dev-volumes/minio_storage
 
 # Only disable DDL generation if database is already initialized
 # (on first run, we need create-tables to bootstrap the schema)

src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java

@@ -1331,8 +1331,8 @@ public void removeTempTag() throws IOException {
             if (e.getCause() instanceof S3Exception) {
                 S3Exception s3e = (S3Exception) e.getCause();
                 if (s3e.statusCode() == 501) {
-                    // In this case, it's likely that tags are not implemented at all (e.g. by
-                    // Minio) so no tag was set either and it's just something to be aware of
+                    // In this case, it's likely that tags are not implemented at all
+                    // so no tag was set either and it's just something to be aware of
                     logger.warning("Temp tag not deleted: Object tags not supported by storage: " + driverId);
                 } else {
                     // In this case, the assumption is that adding tags has worked, so not removing