diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index aa466b913..1ad43a3ad 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,7 +38,7 @@ "ghcr.io/devcontainers/features/go:1": {}, "./rclone": { "rclone_repository": "https://github.com/SwissDataScienceCenter/rclone.git", - "rclone_ref": "v1.69.3+renku-1" + "rclone_ref": "v1.70.0+renku-1" } }, "overrideFeatureInstallOrder": [ diff --git a/components/renku_data_services/data_connectors/api.spec.yaml b/components/renku_data_services/data_connectors/api.spec.yaml index a79d87d4b..f5bc0094c 100644 --- a/components/renku_data_services/data_connectors/api.spec.yaml +++ b/components/renku_data_services/data_connectors/api.spec.yaml @@ -766,7 +766,7 @@ components: type: type: string description: data type of option value. RClone has more options but they map to the ones listed here. - enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"] + enum: ["int", "bool", "string", "stringArray", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"] required: - name - help diff --git a/components/renku_data_services/data_connectors/apispec.py b/components/renku_data_services/data_connectors/apispec.py index 40098ca9a..895a5bef4 100644 --- a/components/renku_data_services/data_connectors/apispec.py +++ b/components/renku_data_services/data_connectors/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2025-04-24T09:27:48+00:00 +# timestamp: 2025-06-19T07:18:06+00:00 from __future__ import annotations @@ -25,6 +25,7 @@ class Type(Enum): int = "int" bool = "bool" string = "string" + stringArray = "stringArray" Time = "Time" Duration = "Duration" MultiEncoder = "MultiEncoder" diff --git a/components/renku_data_services/storage/api.spec.yaml b/components/renku_data_services/storage/api.spec.yaml index 9dfc2fc5d..467f6ebf4 100644 --- a/components/renku_data_services/storage/api.spec.yaml +++ b/components/renku_data_services/storage/api.spec.yaml @@ -431,7 +431,7 @@ components: type: type: string description: data type of option value. RClone has more options but they map to the ones listed here. - enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"] + enum: ["int", "bool", "string", "stringArray", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"] required: - name - help diff --git a/components/renku_data_services/storage/apispec.py b/components/renku_data_services/storage/apispec.py index 79992a0f5..a5b7a1465 100644 --- a/components/renku_data_services/storage/apispec.py +++ b/components/renku_data_services/storage/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2025-04-17T08:59:31+00:00 +# timestamp: 2025-06-19T07:18:05+00:00 from __future__ import annotations @@ -30,6 +30,7 @@ class Type(Enum): int = "int" bool = "bool" string = "string" + stringArray = "stringArray" Time = "Time" Duration = "Duration" MultiEncoder = "MultiEncoder" diff --git a/components/renku_data_services/storage/rclone_schema.autogenerated.json b/components/renku_data_services/storage/rclone_schema.autogenerated.json index 07df87324..6ffcf0576 100644 --- a/components/renku_data_services/storage/rclone_schema.autogenerated.json +++ b/components/renku_data_services/storage/rclone_schema.autogenerated.json @@ -457,6 +457,57 @@ "ValueStr": "16", "Type": "int" }, + { + "Name": "copy_cutoff", + "FieldName": "", + "Help": "Cutoff for switching to multipart copy.\n\nAny files larger than this that need to be server-side copied will be\ncopied in chunks of chunk_size using the put block list API.\n\nFiles smaller than this limit will be copied with the Copy Blob API.", + "Default": 8388608, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "8Mi", + "ValueStr": "8Mi", + "Type": "SizeSuffix" + }, + { + "Name": "copy_concurrency", + "FieldName": "", + "Help": "Concurrency for multipart copy.\n\nThis is the number of chunks of the same file that are copied\nconcurrently.\n\nThese chunks are not buffered in memory and Microsoft recommends\nsetting this value to greater than 1000 in the azcopy documentation.\n\nhttps://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-optimize#increase-concurrency\n\nIn tests, copy speed increases almost linearly with copy\nconcurrency.", + "Default": 512, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "512", + "ValueStr": "512", + "Type": "int" + }, + { + "Name": "use_copy_blob", + "FieldName": "", + "Help": "Whether to use the Copy Blob API when copying to the same storage account.\n\nIf true (the default) then rclone will use the Copy Blob API for\ncopies to the same storage account even when the size is above the\ncopy_cutoff.\n\nRclone assumes that the same storage account means the same config\nand does not check for the same storage account in different configs.\n\nThere should be no need to change this value.\n", + "Default": true, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "true", + "ValueStr": "true", + "Type": "bool" + }, { "Name": "list_chunk", "FieldName": "", @@ -1040,6 +1091,40 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "disable_instance_discovery", + "FieldName": "", + "Help": "Skip requesting Microsoft Entra instance metadata\nThis should be set true only by applications authenticating in\ndisconnected clouds, or private clouds such as Azure Stack.\nIt determines whether rclone requests Microsoft Entra instance\nmetadata from `https://login.microsoft.com/` before\nauthenticating.\nSetting this to true will skip this request, making you responsible\nfor ensuring the configured authority is valid and trustworthy.\n", + "Default": false, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "false", + "ValueStr": "false", + "Type": "bool" + }, + { + "Name": "use_az", + "FieldName": "", + "Help": "Use Azure CLI tool az for authentication\nSet to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)\nas the sole means of authentication.\nSetting this can be useful if you wish to use the az CLI on a host with\na System Managed Identity that you do not want to use.\nDon't set env_auth at the same time.\n", + "Default": false, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "false", + "ValueStr": "false", + "Type": "bool" + }, { "Name": "endpoint", "FieldName": "", @@ -2603,6 +2688,95 @@ "ValueStr": "0s", "Type": "Duration" }, + { + "Name": "adjust_media_files_extensions", + "FieldName": "", + "Help": "Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems", + "Default": true, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "true", + "ValueStr": "true", + "Type": "bool" + }, + { + "Name": "media_extensions", + "FieldName": "", + "Help": "Cloudinary supported media extensions", + "Default": [ + "3ds", + "3g2", + "3gp", + "ai", + "arw", + "avi", + "avif", + "bmp", + "bw", + "cr2", + "cr3", + "djvu", + "dng", + "eps3", + "fbx", + "flif", + "flv", + "gif", + "glb", + "gltf", + "hdp", + "heic", + "heif", + "ico", + "indd", + "jp2", + "jpe", + "jpeg", + "jpg", + "jxl", + "jxr", + "m2ts", + "mov", + "mp4", + "mpeg", + "mts", + "mxf", + "obj", + "ogv", + "pdf", + "ply", + "png", + "psd", + "svg", + "tga", + "tif", + "tiff", + "ts", + "u3ma", + "usdz", + "wdp", + "webm", + "webp", + "wmv" + ], + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "[3ds 3g2 3gp ai arw avi avif bmp bw cr2 cr3 djvu dng eps3 fbx flif flv gif glb gltf hdp heic heif ico indd jp2 jpe jpeg jpg jxl jxr m2ts mov mp4 mpeg mts mxf obj ogv pdf ply png psd svg tga tif tiff ts u3ma usdz wdp webm webp wmv]", + "ValueStr": "3ds,3g2,3gp,ai,arw,avi,avif,bmp,bw,cr2,cr3,djvu,dng,eps3,fbx,flif,flv,gif,glb,gltf,hdp,heic,heif,ico,indd,jp2,jpe,jpeg,jpg,jxl,jxr,m2ts,mov,mp4,mpeg,mts,mxf,obj,ogv,pdf,ply,png,psd,svg,tga,tif,tiff,ts,u3ma,usdz,wdp,webm,webp,wmv", + "Type": "stringArray" + }, { "Name": "description", "FieldName": "", @@ -3132,6 +3306,23 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "doi_resolver_api_url", + "FieldName": "", + "Help": "The URL of the DOI resolver API to use.\n\nThe DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used.\n\nDefaults to \"https://doi.org/api\".", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, { "Name": "description", "FieldName": "", @@ -4184,6 +4375,12 @@ "Long": "This command copies files by ID\n\nUsage:\n\n rclone backend copyid drive: ID path\n rclone backend copyid drive: ID1 path1 ID2 path2\n\nIt copies the drive file with ID given to the path (an rclone path which\nwill be passed internally to rclone copyto). The ID and path pairs can be\nrepeated.\n\nThe path should end with a / to indicate copy the file as named to\nthis directory. If it doesn't end with a / then the last path\ncomponent will be used as the file name.\n\nIf the destination is a drive backend then server-side copying will be\nattempted if possible.\n\nUse the --interactive/-i or --dry-run flag to see what would be copied before copying.\n", "Opts": null }, + { + "Name": "moveid", + "Short": "Move files by ID", + "Long": "This command moves files by ID\n\nUsage:\n\n rclone backend moveid drive: ID path\n rclone backend moveid drive: ID1 path1 ID2 path2\n\nIt moves the drive file with ID given to the path (an rclone path which\nwill be passed internally to rclone moveto).\n\nThe path should end with a / to indicate move the file as named to\nthis directory. If it doesn't end with a / then the last path\ncomponent will be used as the file name.\n\nIf the destination is a drive backend then server-side moving will be\nattempted if possible.\n\nUse the --interactive/-i or --dry-run flag to see what would be moved beforehand.\n", + "Opts": null + }, { "Name": "exportformats", "Short": "Dump the export formats for debug purposes", @@ -4515,6 +4712,60 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "export_formats", + "FieldName": "", + "Help": "Comma separated list of preferred formats for exporting files\n\nCertain Dropbox files can only be accessed by exporting them to another format.\nThese include Dropbox Paper documents.\n\nFor each such file, rclone will choose the first format on this list that Dropbox\nconsiders valid. If none is valid, it will choose Dropbox's default format.\n\nKnown formats include: \"html\", \"md\" (markdown)", + "Default": [ + "html", + "md" + ], + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "html,md", + "ValueStr": "html,md", + "Type": "CommaSepList" + }, + { + "Name": "skip_exports", + "FieldName": "", + "Help": "Skip exportable files in all listings.\n\nIf given, exportable files practically become invisible to rclone.", + "Default": false, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "false", + "ValueStr": "false", + "Type": "bool" + }, + { + "Name": "show_all_exports", + "FieldName": "", + "Help": "Show all exportable files in listings.\n\nAdding this flag will allow all exportable files to be server side copied.\nNote that rclone doesn't add extensions to the exportable file names in this mode.\n\nDo **not** use this flag when trying to download exportable files - rclone\nwill fail to download them.\n", + "Default": false, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "false", + "ValueStr": "false", + "Type": "bool" + }, { "Name": "batch_mode", "FieldName": "", @@ -4569,10 +4820,10 @@ { "Name": "batch_commit_timeout", "FieldName": "", - "Help": "Max time to wait for a batch to finish committing", + "Help": "Max time to wait for a batch to finish committing. (no longer used)", "Default": 600000000000, "Value": null, - "Hide": 0, + "Hide": 3, "Required": false, "IsPassword": false, "NoPrefix": false, @@ -4897,6 +5148,68 @@ "Hide": false, "MetadataInfo": null }, + { + "Name": "filelu", + "Description": "FileLu Cloud Storage", + "Prefix": "filelu", + "Options": [ + { + "Name": "key", + "FieldName": "", + "Help": "Your FileLu Rclone key from My Account", + "Default": "", + "Value": null, + "Hide": 0, + "Required": true, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": true, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, + { + "Name": "encoding", + "FieldName": "", + "Help": "The encoding for the backend.\n\nSee the [encoding section in the overview](/overview/#encoding) for more info.", + "Default": 536870910, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,Colon,Question,Asterisk,Pipe,Hash,Percent,BackSlash,CrLf,Del,Ctl,LeftSpace,LeftPeriod,LeftTilde,LeftCrLfHtVt,RightSpace,RightPeriod,RightCrLfHtVt,InvalidUtf8,Dot,SquareBracket,Semicolon,Exclamation", + "ValueStr": "Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,Colon,Question,Asterisk,Pipe,Hash,Percent,BackSlash,CrLf,Del,Ctl,LeftSpace,LeftPeriod,LeftTilde,LeftCrLfHtVt,RightSpace,RightPeriod,RightCrLfHtVt,InvalidUtf8,Dot,SquareBracket,Semicolon,Exclamation", + "Type": "Encoding" + }, + { + "Name": "description", + "FieldName": "", + "Help": "Description of the remote.", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + } + ], + "CommandHelp": null, + "Aliases": null, + "Hide": false, + "MetadataInfo": null + }, { "Name": "filescom", "Description": "Files.com", @@ -5355,6 +5668,23 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "http_proxy", + "FieldName": "", + "Help": "URL for HTTP CONNECT proxy\n\nSet this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.\n", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, { "Name": "no_check_upload", "FieldName": "", @@ -6370,7 +6700,7 @@ { "Name": "proxy", "FieldName": "", - "Help": "Use the gphotosdl proxy for downloading the full resolution images\n\nThe Google API will deliver images and video which aren't full\nresolution, and/or have EXIF data missing.\n\nHowever if you ue the gphotosdl proxy tnen you can download original,\nunchanged images.\n\nThis runs a headless browser in the background.\n\nDownload the software from [gphotosdl](https://github.com/rclone/gphotosdl)\n\nFirst run with\n\n gphotosdl -login\n\nThen once you have logged into google photos close the browser window\nand run\n\n gphotosdl\n\nThen supply the parameter `--gphotos-proxy \"http://localhost:8282\"` to make\nrclone use the proxy.\n", + "Help": "Use the gphotosdl proxy for downloading the full resolution images\n\nThe Google API will deliver images and video which aren't full\nresolution, and/or have EXIF data missing.\n\nHowever if you use the gphotosdl proxy then you can download original,\nunchanged images.\n\nThis runs a headless browser in the background.\n\nDownload the software from [gphotosdl](https://github.com/rclone/gphotosdl)\n\nFirst run with\n\n gphotosdl -login\n\nThen once you have logged into google photos close the browser window\nand run\n\n gphotosdl\n\nThen supply the parameter `--gphotos-proxy \"http://localhost:8282\"` to make\nrclone use the proxy.\n", "Default": "", "Value": null, "Hide": 0, @@ -6455,10 +6785,10 @@ { "Name": "batch_commit_timeout", "FieldName": "", - "Help": "Max time to wait for a batch to finish committing", + "Help": "Max time to wait for a batch to finish committing. (no longer used)", "Default": 600000000000, "Value": null, - "Hide": 0, + "Hide": 3, "Required": false, "IsPassword": false, "NoPrefix": false, @@ -7611,6 +7941,40 @@ "ValueStr": "https://archive.org", "Type": "string" }, + { + "Name": "item_metadata", + "FieldName": "", + "Help": "Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set.\nFormat is key=value and the 'x-archive-meta-' prefix is automatically added.", + "Default": [], + "Value": null, + "Hide": 2, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "[]", + "ValueStr": "", + "Type": "stringArray" + }, + { + "Name": "item_derive", + "FieldName": "", + "Help": "Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload.\nThe derive process produces a number of secondary files from an upload to make an upload more usable on the web.\nSetting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure.", + "Default": true, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "true", + "ValueStr": "true", + "Type": "bool" + }, { "Name": "disable_checksum", "FieldName": "", @@ -9444,6 +9808,23 @@ "ValueStr": "global", "Type": "string" }, + { + "Name": "upload_cutoff", + "FieldName": "", + "Help": "Cutoff for switching to chunked upload.\n\nAny files larger than this will be uploaded in chunks of chunk_size.\n\nThis is disabled by default as uploading using single part uploads\ncauses rclone to use twice the storage on Onedrive business as when\nrclone sets the modification time after the upload Onedrive creates a\nnew version.\n\nSee: https://github.com/rclone/rclone/issues/1716\n", + "Default": -1, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "off", + "ValueStr": "off", + "Type": "SizeSuffix" + }, { "Name": "chunk_size", "FieldName": "", @@ -10079,6 +10460,37 @@ "ValueStr": "10Mi", "Type": "SizeSuffix" }, + { + "Name": "access", + "FieldName": "", + "Help": "Files and folders will be uploaded with this access permission (default private)", + "Default": "private", + "Value": null, + "Examples": [ + { + "Value": "private", + "Help": "The file or folder access can be granted in a way that will allow select users to view, read or write what is absolutely essential for them." + }, + { + "Value": "public", + "Help": "The file or folder can be downloaded by anyone from a web browser. The link can be shared in any way," + }, + { + "Value": "hidden", + "Help": "The file or folder can be accessed has the same restrictions as Public if the user knows the URL of the file or folder link in order to access the contents" + } + ], + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "private", + "ValueStr": "private", + "Type": "string" + }, { "Name": "description", "FieldName": "", @@ -12175,7 +12587,7 @@ }, { "Name": "s3", - "Description": "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others", + "Description": "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others", "Prefix": "s3", "Options": [ { @@ -12217,6 +12629,14 @@ "Value": "Dreamhost", "Help": "Dreamhost DreamObjects" }, + { + "Value": "Exaba", + "Help": "Exaba Object Storage" + }, + { + "Value": "FlashBlade", + "Help": "Pure Storage FlashBlade Object Storage" + }, { "Value": "GCS", "Help": "Google Cloud Storage" @@ -12257,6 +12677,10 @@ "Value": "Magalu", "Help": "Magalu Object Storage" }, + { + "Value": "Mega", + "Help": "MEGA S4 Object Storage" + }, { "Value": "Minio", "Help": "Minio Object Storage" @@ -12978,7 +13402,7 @@ "Name": "region", "FieldName": "", "Help": "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - "Provider": "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive", + "Provider": "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega", "Default": "", "Value": null, "Examples": [ @@ -13684,6 +14108,24 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "endpoint", + "FieldName": "", + "Help": "Endpoint for Lyve Cloud S3 API.\nRequired when using an S3 clone. Please type in your LyveCloud endpoint.\nExamples:\n- s3.us-west-1.{account_name}.lyve.seagate.com (US West 1 - California)\n- s3.eu-west-1.{account_name}.lyve.seagate.com (EU West 1 - Ireland)", + "Provider": "LyveCloud", + "Default": "", + "Value": null, + "Hide": 0, + "Required": true, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, { "Name": "endpoint", "FieldName": "", @@ -14332,7 +14774,7 @@ "Name": "endpoint", "FieldName": "", "Help": "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - "Provider": "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox", + "Provider": "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox", "Default": "", "Value": null, "Examples": [ @@ -14396,21 +14838,6 @@ "Help": "SeaweedFS S3 localhost", "Provider": "SeaweedFS" }, - { - "Value": "s3.us-east-1.lyvecloud.seagate.com", - "Help": "Seagate Lyve Cloud US East 1 (Virginia)", - "Provider": "LyveCloud" - }, - { - "Value": "s3.us-west-1.lyvecloud.seagate.com", - "Help": "Seagate Lyve Cloud US West 1 (California)", - "Provider": "LyveCloud" - }, - { - "Value": "s3.ap-southeast-1.lyvecloud.seagate.com", - "Help": "Seagate Lyve Cloud AP Southeast 1 (Singapore)", - "Provider": "LyveCloud" - }, { "Value": "oos.eu-west-2.outscale.com", "Help": "Outscale EU West 2 (Paris)", @@ -14520,6 +14947,26 @@ "Value": "s3.ir-tbz-sh1.arvanstorage.ir", "Help": "ArvanCloud Tabriz Iran (Shahriar) endpoint", "Provider": "ArvanCloud" + }, + { + "Value": "s3.eu-central-1.s4.mega.io", + "Help": "Mega S4 eu-central-1 (Amsterdam)", + "Provider": "Mega" + }, + { + "Value": "s3.eu-central-2.s4.mega.io", + "Help": "Mega S4 eu-central-2 (Bettembourg)", + "Provider": "Mega" + }, + { + "Value": "s3.ca-central-1.s4.mega.io", + "Help": "Mega S4 ca-central-1 (Montreal)", + "Provider": "Mega" + }, + { + "Value": "s3.ca-west-1.s4.mega.io", + "Help": "Mega S4 ca-west-1 (Vancouver)", + "Provider": "Mega" } ], "Hide": 0, @@ -15121,7 +15568,7 @@ "Name": "location_constraint", "FieldName": "", "Help": "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - "Provider": "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox", + "Provider": "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,FlashBlade,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox,Mega", "Default": "", "Value": null, "Hide": 0, @@ -15139,7 +15586,7 @@ "Name": "acl", "FieldName": "", "Help": "Canned ACL used when creating buckets and storing or copying objects.\n\nThis ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.\n\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl\n\nNote that this ACL is applied when server-side copying objects as S3\ndoesn't copy the ACL from the source but rather writes a fresh one.\n\nIf the acl is an empty string then no X-Amz-Acl: header is added and\nthe default (private) will be used.\n", - "Provider": "!Storj,Selectel,Synology,Cloudflare", + "Provider": "!Storj,Selectel,Synology,Cloudflare,FlashBlade,Mega", "Default": "", "Value": null, "Examples": [ @@ -15214,6 +15661,7 @@ "Name": "bucket_acl", "FieldName": "", "Help": "Canned ACL used when creating buckets.\n\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl\n\nNote that this ACL is applied when only when creating buckets. If it\nisn't set then \"acl\" is used instead.\n\nIf the \"acl\" and \"bucket_acl\" are empty strings then no X-Amz-Acl:\nheader is added and the default (private) will be used.\n", + "Provider": "!Storj,Selectel,Synology,Cloudflare,FlashBlade", "Default": "", "Value": null, "Examples": [ @@ -16423,6 +16871,46 @@ "ValueStr": "unset", "Type": "Tristate" }, + { + "Name": "use_x_id", + "FieldName": "", + "Help": "Set if rclone should add x-id URL parameters.\n\nYou can change this if you want to disable the AWS SDK from\nadding x-id URL parameters.\n\nThis shouldn't be necessary in normal operation.\n\nThis should be automatically set correctly for all providers rclone\nknows about - please make a bug report if not.\n", + "Default": { + "Value": false, + "Valid": false + }, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "unset", + "ValueStr": "unset", + "Type": "Tristate" + }, + { + "Name": "sign_accept_encoding", + "FieldName": "", + "Help": "Set if rclone should include Accept-Encoding as part of the signature.\n\nYou can change this if you want to stop rclone including\nAccept-Encoding as part of the signature.\n\nThis shouldn't be necessary in normal operation.\n\nThis should be automatically set correctly for all providers rclone\nknows about - please make a bug report if not.\n", + "Default": { + "Value": false, + "Valid": false + }, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "unset", + "ValueStr": "unset", + "Type": "Tristate" + }, { "Name": "directory_bucket", "FieldName": "", @@ -16458,6 +16946,42 @@ "ValueStr": "Off", "Type": "Bits" }, + { + "Name": "ibm_api_key", + "FieldName": "", + "Help": "IBM API Key to be used to obtain IAM token", + "Provider": "IBMCOS", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, + { + "Name": "ibm_resource_instance_id", + "FieldName": "", + "Help": "IBM service instance id", + "Provider": "IBMCOS", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, { "Name": "description", "FieldName": "", @@ -17422,6 +17946,23 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "http_proxy", + "FieldName": "", + "Help": "URL for HTTP CONNECT proxy\n\nSet this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.\n", + "Default": "", + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": true, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "", + "ValueStr": "", + "Type": "string" + }, { "Name": "copy_is_hardlink", "FieldName": "", @@ -17902,6 +18443,23 @@ "ValueStr": "", "Type": "string" }, + { + "Name": "use_kerberos", + "FieldName": "", + "Help": "Use Kerberos authentication.\n\nIf set, rclone will use Kerberos authentication instead of NTLM. This\nrequires a valid Kerberos configuration and credentials cache to be\navailable, either in the default locations or as specified by the\nKRB5_CONFIG and KRB5CCNAME environment variables.\n", + "Default": false, + "Value": null, + "Hide": 0, + "Required": false, + "IsPassword": false, + "NoPrefix": false, + "Advanced": false, + "Exclusive": false, + "Sensitive": false, + "DefaultStr": "false", + "ValueStr": "false", + "Type": "bool" + }, { "Name": "idle_timeout", "FieldName": "", @@ -19416,7 +19974,11 @@ }, { "Value": "owncloud", - "Help": "Owncloud" + "Help": "Owncloud 10 PHP based WebDAV server" + }, + { + "Value": "infinitescale", + "Help": "ownCloud Infinite Scale" }, { "Value": "sharepoint", diff --git a/flake.nix b/flake.nix index 657906295..441fbfea7 100644 --- a/flake.nix +++ b/flake.nix @@ -61,12 +61,12 @@ devshellToolsPkgs = devshell-tools.packages.${system}; rclone = pkgs.rclone.overrideAttrs (old: { - version = "1.69.3"; + version = "1.70.0"; vendorHash = "sha256-Wu9d98SIENCkJYoGT/f9KN8vnYYGMN7HxhzqtkOYQ/8="; src = pkgs.fetchFromGitHub { owner = "SwissDataScienceCenter"; repo = "rclone"; - rev = "v1.69.3+renku-1"; + rev = "v1.70.0+renku-1"; sha256 = "sha256-aorgWwYBVVOYhMXXBDWBMXkaZi0WjnGaMoRlwXCa5w4="; }; }); diff --git a/projects/renku_data_service/Dockerfile b/projects/renku_data_service/Dockerfile index 0112de2b1..bdb0cc270 100644 --- a/projects/renku_data_service/Dockerfile +++ b/projects/renku_data_service/Dockerfile @@ -1,5 +1,5 @@ ARG RCLONE_IMAGE_REPOSITORY="ghcr.io/swissdatasciencecenter/rclone" -ARG RCLONE_IMAGE_TAG="sha-9cbde11" +ARG RCLONE_IMAGE_TAG="sha-66b7f3e" FROM ${RCLONE_IMAGE_REPOSITORY}:${RCLONE_IMAGE_TAG} AS rclone FROM python:3.13-bookworm AS builder diff --git a/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr b/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr index 8d73b661e..e2c9ff941 100644 --- a/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr +++ b/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr @@ -702,6 +702,74 @@ 'sensitive': False, 'type': 'int', }), + dict({ + 'advanced': True, + 'default': 8388608.0, + 'default_str': '8Mi', + 'exclusive': False, + 'help': ''' + Cutoff for switching to multipart copy. + + Any files larger than this that need to be server-side copied will be + copied in chunks of chunk_size using the put block list API. + + Files smaller than this limit will be copied with the Copy Blob API. + ''', + 'ispassword': False, + 'name': 'copy_cutoff', + 'required': False, + 'sensitive': False, + 'type': 'SizeSuffix', + }), + dict({ + 'advanced': True, + 'default': 512.0, + 'default_str': '512', + 'exclusive': False, + 'help': ''' + Concurrency for multipart copy. + + This is the number of chunks of the same file that are copied + concurrently. + + These chunks are not buffered in memory and Microsoft recommends + setting this value to greater than 1000 in the azcopy documentation. + + https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-optimize#increase-concurrency + + In tests, copy speed increases almost linearly with copy + concurrency. + ''', + 'ispassword': False, + 'name': 'copy_concurrency', + 'required': False, + 'sensitive': False, + 'type': 'int', + }), + dict({ + 'advanced': True, + 'default': True, + 'default_str': 'true', + 'exclusive': False, + 'help': ''' + Whether to use the Copy Blob API when copying to the same storage account. + + If true (the default) then rclone will use the Copy Blob API for + copies to the same storage account even when the size is above the + copy_cutoff. + + Rclone assumes that the same storage account means the same config + and does not check for the same storage account in different configs. + + There should be no need to change this value. + + ''', + 'ispassword': False, + 'name': 'use_copy_blob', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), dict({ 'advanced': True, 'default': 5000.0, @@ -1314,6 +1382,48 @@ 'sensitive': True, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': False, + 'default_str': 'false', + 'exclusive': False, + 'help': ''' + Skip requesting Microsoft Entra instance metadata + This should be set true only by applications authenticating in + disconnected clouds, or private clouds such as Azure Stack. + It determines whether rclone requests Microsoft Entra instance + metadata from `https://login.microsoft.com/` before + authenticating. + Setting this to true will skip this request, making you responsible + for ensuring the configured authority is valid and trustworthy. + + ''', + 'ispassword': False, + 'name': 'disable_instance_discovery', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), + dict({ + 'advanced': True, + 'default': False, + 'default_str': 'false', + 'exclusive': False, + 'help': ''' + Use Azure CLI tool az for authentication + Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/) + as the sole means of authentication. + Setting this can be useful if you wish to use the az CLI on a host with + a System Managed Identity that you do not want to use. + Don't set env_auth at the same time. + + ''', + 'ispassword': False, + 'name': 'use_az', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), dict({ 'advanced': True, 'default': '', @@ -2108,6 +2218,85 @@ 'sensitive': False, 'type': 'Duration', }), + dict({ + 'advanced': True, + 'default': True, + 'default_str': 'true', + 'exclusive': False, + 'help': 'Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems', + 'ispassword': False, + 'name': 'adjust_media_files_extensions', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), + dict({ + 'advanced': True, + 'default': list([ + '3ds', + '3g2', + '3gp', + 'ai', + 'arw', + 'avi', + 'avif', + 'bmp', + 'bw', + 'cr2', + 'cr3', + 'djvu', + 'dng', + 'eps3', + 'fbx', + 'flif', + 'flv', + 'gif', + 'glb', + 'gltf', + 'hdp', + 'heic', + 'heif', + 'ico', + 'indd', + 'jp2', + 'jpe', + 'jpeg', + 'jpg', + 'jxl', + 'jxr', + 'm2ts', + 'mov', + 'mp4', + 'mpeg', + 'mts', + 'mxf', + 'obj', + 'ogv', + 'pdf', + 'ply', + 'png', + 'psd', + 'svg', + 'tga', + 'tif', + 'tiff', + 'ts', + 'u3ma', + 'usdz', + 'wdp', + 'webm', + 'webp', + 'wmv', + ]), + 'default_str': '[3ds 3g2 3gp ai arw avi avif bmp bw cr2 cr3 djvu dng eps3 fbx flif flv gif glb gltf hdp heic heif ico indd jp2 jpe jpeg jpg jxl jxr m2ts mov mp4 mpeg mts mxf obj ogv pdf ply png psd svg tga tif tiff ts u3ma usdz wdp webm webp wmv]', + 'exclusive': False, + 'help': 'Cloudinary supported media extensions', + 'ispassword': False, + 'name': 'media_extensions', + 'required': False, + 'sensitive': False, + 'type': 'stringArray', + }), dict({ 'advanced': True, 'default': '', @@ -2173,6 +2362,24 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': ''' + The URL of the DOI resolver API to use. + + The DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used. + + Defaults to "https://doi.org/api". + ''', + 'ispassword': False, + 'name': 'doi_resolver_api_url', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': True, 'default': '', @@ -3389,6 +3596,68 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': list([ + 'html', + 'md', + ]), + 'default_str': 'html,md', + 'exclusive': False, + 'help': ''' + Comma separated list of preferred formats for exporting files + + Certain Dropbox files can only be accessed by exporting them to another format. + These include Dropbox Paper documents. + + For each such file, rclone will choose the first format on this list that Dropbox + considers valid. If none is valid, it will choose Dropbox's default format. + + Known formats include: "html", "md" (markdown) + ''', + 'ispassword': False, + 'name': 'export_formats', + 'required': False, + 'sensitive': False, + 'type': 'CommaSepList', + }), + dict({ + 'advanced': True, + 'default': False, + 'default_str': 'false', + 'exclusive': False, + 'help': ''' + Skip exportable files in all listings. + + If given, exportable files practically become invisible to rclone. + ''', + 'ispassword': False, + 'name': 'skip_exports', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), + dict({ + 'advanced': True, + 'default': False, + 'default_str': 'false', + 'exclusive': False, + 'help': ''' + Show all exportable files in listings. + + Adding this flag will allow all exportable files to be server side copied. + Note that rclone doesn't add extensions to the exportable file names in this mode. + + Do **not** use this flag when trying to download exportable files - rclone + will fail to download them. + + ''', + 'ispassword': False, + 'name': 'show_all_exports', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), dict({ 'advanced': True, 'default': 'sync', @@ -3478,7 +3747,7 @@ 'default': 600000000000.0, 'default_str': '10m0s', 'exclusive': False, - 'help': 'Max time to wait for a batch to finish committing', + 'help': 'Max time to wait for a batch to finish committing. (no longer used)', 'ispassword': False, 'name': 'batch_commit_timeout', 'required': False, @@ -3753,6 +4022,53 @@ ]), 'prefix': 'filefabric', }), + dict({ + 'description': 'FileLu Cloud Storage', + 'name': 'filelu', + 'options': list([ + dict({ + 'advanced': False, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': 'Your FileLu Rclone key from My Account', + 'ispassword': False, + 'name': 'key', + 'required': True, + 'sensitive': True, + 'type': 'string', + }), + dict({ + 'advanced': True, + 'default': 536870910.0, + 'default_str': 'Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,Colon,Question,Asterisk,Pipe,Hash,Percent,BackSlash,CrLf,Del,Ctl,LeftSpace,LeftPeriod,LeftTilde,LeftCrLfHtVt,RightSpace,RightPeriod,RightCrLfHtVt,InvalidUtf8,Dot,SquareBracket,Semicolon,Exclamation', + 'exclusive': False, + 'help': ''' + The encoding for the backend. + + See the [encoding section in the overview](/overview/#encoding) for more info. + ''', + 'ispassword': False, + 'name': 'encoding', + 'required': False, + 'sensitive': False, + 'type': 'Encoding', + }), + dict({ + 'advanced': True, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': 'Description of the remote.', + 'ispassword': False, + 'name': 'description', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), + ]), + 'prefix': 'filelu', + }), dict({ 'description': 'Files.com', 'name': 'filescom', @@ -4141,6 +4457,23 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': ''' + URL for HTTP CONNECT proxy + + Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb. + + ''', + 'ispassword': False, + 'name': 'http_proxy', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': True, 'default': False, @@ -5085,7 +5418,7 @@ The Google API will deliver images and video which aren't full resolution, and/or have EXIF data missing. - However if you ue the gphotosdl proxy tnen you can download original, + However if you use the gphotosdl proxy then you can download original, unchanged images. This runs a headless browser in the background. @@ -5214,7 +5547,7 @@ 'default': 600000000000.0, 'default_str': '10m0s', 'exclusive': False, - 'help': 'Max time to wait for a batch to finish committing', + 'help': 'Max time to wait for a batch to finish committing. (no longer used)', 'ispassword': False, 'name': 'batch_commit_timeout', 'required': False, @@ -6022,6 +6355,38 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': list([ + ]), + 'default_str': '[]', + 'exclusive': False, + 'help': ''' + Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set. + Format is key=value and the 'x-archive-meta-' prefix is automatically added. + ''', + 'ispassword': False, + 'name': 'item_metadata', + 'required': False, + 'sensitive': False, + 'type': 'stringArray', + }), + dict({ + 'advanced': False, + 'default': True, + 'default_str': 'true', + 'exclusive': False, + 'help': ''' + Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload. + The derive process produces a number of secondary files from an upload to make an upload more usable on the web. + Setting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure. + ''', + 'ispassword': False, + 'name': 'item_derive', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), dict({ 'advanced': True, 'default': True, @@ -7039,6 +7404,30 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': -1.0, + 'default_str': 'off', + 'exclusive': False, + 'help': ''' + Cutoff for switching to chunked upload. + + Any files larger than this will be uploaded in chunks of chunk_size. + + This is disabled by default as uploading using single part uploads + causes rclone to use twice the storage on Onedrive business as when + rclone sets the modification time after the upload Onedrive creates a + new version. + + See: https://github.com/rclone/rclone/issues/1716 + + ''', + 'ispassword': False, + 'name': 'upload_cutoff', + 'required': False, + 'sensitive': False, + 'type': 'SizeSuffix', + }), dict({ 'advanced': True, 'default': 10485760.0, @@ -7609,6 +7998,32 @@ 'sensitive': False, 'type': 'SizeSuffix', }), + dict({ + 'advanced': True, + 'default': 'private', + 'default_str': 'private', + 'examples': list([ + dict({ + 'help': 'The file or folder access can be granted in a way that will allow select users to view, read or write what is absolutely essential for them.', + 'value': 'private', + }), + dict({ + 'help': 'The file or folder can be downloaded by anyone from a web browser. The link can be shared in any way,', + 'value': 'public', + }), + dict({ + 'help': 'The file or folder can be accessed has the same restrictions as Public if the user knows the URL of the file or folder link in order to access the contents', + 'value': 'hidden', + }), + ]), + 'exclusive': False, + 'help': 'Files and folders will be uploaded with this access permission (default private)', + 'ispassword': False, + 'name': 'access', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': True, 'default': '', @@ -9432,7 +9847,7 @@ 'prefix': 'quatrix', }), dict({ - 'description': 'Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others', + 'description': 'Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others', 'name': 's3', 'options': list([ dict({ @@ -9472,6 +9887,14 @@ 'help': 'Dreamhost DreamObjects', 'value': 'Dreamhost', }), + dict({ + 'help': 'Exaba Object Storage', + 'value': 'Exaba', + }), + dict({ + 'help': 'Pure Storage FlashBlade Object Storage', + 'value': 'FlashBlade', + }), dict({ 'help': 'Google Cloud Storage', 'value': 'GCS', @@ -9512,6 +9935,10 @@ 'help': 'Magalu Object Storage', 'value': 'Magalu', }), + dict({ + 'help': 'MEGA S4 Object Storage', + 'value': 'Mega', + }), dict({ 'help': 'Minio Object Storage', 'value': 'Minio', @@ -10323,7 +10750,7 @@ ''', 'ispassword': False, 'name': 'region', - 'provider': '!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive', + 'provider': '!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega', 'required': False, 'sensitive': False, 'type': 'string', @@ -10993,6 +11420,25 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': False, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': ''' + Endpoint for Lyve Cloud S3 API. + Required when using an S3 clone. Please type in your LyveCloud endpoint. + Examples: + - s3.us-west-1.{account_name}.lyve.seagate.com (US West 1 - California) + - s3.eu-west-1.{account_name}.lyve.seagate.com (EU West 1 - Ireland) + ''', + 'ispassword': False, + 'name': 'endpoint', + 'provider': 'LyveCloud', + 'required': True, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': False, 'default': '', @@ -11642,21 +12088,6 @@ 'provider': 'SeaweedFS', 'value': 'localhost:8333', }), - dict({ - 'help': 'Seagate Lyve Cloud US East 1 (Virginia)', - 'provider': 'LyveCloud', - 'value': 's3.us-east-1.lyvecloud.seagate.com', - }), - dict({ - 'help': 'Seagate Lyve Cloud US West 1 (California)', - 'provider': 'LyveCloud', - 'value': 's3.us-west-1.lyvecloud.seagate.com', - }), - dict({ - 'help': 'Seagate Lyve Cloud AP Southeast 1 (Singapore)', - 'provider': 'LyveCloud', - 'value': 's3.ap-southeast-1.lyvecloud.seagate.com', - }), dict({ 'help': 'Outscale EU West 2 (Paris)', 'provider': 'Outscale', @@ -11767,6 +12198,26 @@ 'provider': 'ArvanCloud', 'value': 's3.ir-tbz-sh1.arvanstorage.ir', }), + dict({ + 'help': 'Mega S4 eu-central-1 (Amsterdam)', + 'provider': 'Mega', + 'value': 's3.eu-central-1.s4.mega.io', + }), + dict({ + 'help': 'Mega S4 eu-central-2 (Bettembourg)', + 'provider': 'Mega', + 'value': 's3.eu-central-2.s4.mega.io', + }), + dict({ + 'help': 'Mega S4 ca-central-1 (Montreal)', + 'provider': 'Mega', + 'value': 's3.ca-central-1.s4.mega.io', + }), + dict({ + 'help': 'Mega S4 ca-west-1 (Vancouver)', + 'provider': 'Mega', + 'value': 's3.ca-west-1.s4.mega.io', + }), ]), 'exclusive': False, 'help': ''' @@ -11776,7 +12227,7 @@ ''', 'ispassword': False, 'name': 'endpoint', - 'provider': '!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Switch', + 'provider': '!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Switch', 'required': True, 'sensitive': False, 'type': 'string', @@ -12370,7 +12821,7 @@ ''', 'ispassword': False, 'name': 'location_constraint', - 'provider': '!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox', + 'provider': '!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,FlashBlade,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox,Mega', 'required': False, 'sensitive': False, 'type': 'string', @@ -12493,7 +12944,7 @@ ''', 'ispassword': False, 'name': 'acl', - 'provider': '!Storj,Selectel,Synology,Cloudflare', + 'provider': '!Storj,Selectel,Synology,Cloudflare,FlashBlade,Mega', 'required': False, 'sensitive': False, 'type': 'string', @@ -12548,6 +12999,7 @@ ''', 'ispassword': False, 'name': 'bucket_acl', + 'provider': '!Storj,Selectel,Synology,Cloudflare,FlashBlade', 'required': False, 'sensitive': False, 'type': 'string', @@ -13831,6 +14283,58 @@ 'sensitive': False, 'type': 'Tristate', }), + dict({ + 'advanced': True, + 'default': dict({ + 'valid': False, + 'value': False, + }), + 'default_str': 'unset', + 'exclusive': False, + 'help': ''' + Set if rclone should add x-id URL parameters. + + You can change this if you want to disable the AWS SDK from + adding x-id URL parameters. + + This shouldn't be necessary in normal operation. + + This should be automatically set correctly for all providers rclone + knows about - please make a bug report if not. + + ''', + 'ispassword': False, + 'name': 'use_x_id', + 'required': False, + 'sensitive': False, + 'type': 'Tristate', + }), + dict({ + 'advanced': True, + 'default': dict({ + 'valid': False, + 'value': False, + }), + 'default_str': 'unset', + 'exclusive': False, + 'help': ''' + Set if rclone should include Accept-Encoding as part of the signature. + + You can change this if you want to stop rclone including + Accept-Encoding as part of the signature. + + This shouldn't be necessary in normal operation. + + This should be automatically set correctly for all providers rclone + knows about - please make a bug report if not. + + ''', + 'ispassword': False, + 'name': 'sign_accept_encoding', + 'required': False, + 'sensitive': False, + 'type': 'Tristate', + }), dict({ 'advanced': True, 'default': False, @@ -13899,6 +14403,32 @@ 'sensitive': False, 'type': 'Bits', }), + dict({ + 'advanced': False, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': 'IBM API Key to be used to obtain IAM token', + 'ispassword': False, + 'name': 'ibm_api_key', + 'provider': 'IBMCOS', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), + dict({ + 'advanced': False, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': 'IBM service instance id', + 'ispassword': False, + 'name': 'ibm_resource_instance_id', + 'provider': 'IBMCOS', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': True, 'default': '', @@ -14875,6 +15405,23 @@ 'sensitive': False, 'type': 'string', }), + dict({ + 'advanced': True, + 'default': '', + 'default_str': '', + 'exclusive': False, + 'help': ''' + URL for HTTP CONNECT proxy + + Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb. + + ''', + 'ispassword': False, + 'name': 'http_proxy', + 'required': False, + 'sensitive': False, + 'type': 'string', + }), dict({ 'advanced': True, 'default': False, @@ -15278,6 +15825,26 @@ 'sensitive': True, 'type': 'string', }), + dict({ + 'advanced': False, + 'default': False, + 'default_str': 'false', + 'exclusive': False, + 'help': ''' + Use Kerberos authentication. + + If set, rclone will use Kerberos authentication instead of NTLM. This + requires a valid Kerberos configuration and credentials cache to be + available, either in the default locations or as specified by the + KRB5_CONFIG and KRB5CCNAME environment variables. + + ''', + 'ispassword': False, + 'name': 'use_kerberos', + 'required': False, + 'sensitive': False, + 'type': 'bool', + }), dict({ 'advanced': True, 'default': 60000000000.0, @@ -16496,9 +17063,13 @@ 'value': 'nextcloud', }), dict({ - 'help': 'Owncloud', + 'help': 'Owncloud 10 PHP based WebDAV server', 'value': 'owncloud', }), + dict({ + 'help': 'ownCloud Infinite Scale', + 'value': 'infinitescale', + }), dict({ 'help': 'Sharepoint Online, authenticated by Microsoft account', 'value': 'sharepoint',