S3: preserve repeated object path slashes with a per-driver option

CHANGES.rst

@@ -76,6 +76,11 @@ Compute
 Storage
 ~~~~~~~
 
+- [S3] Add ``ex_allow_path_double_slashes`` to preserve object key slashes
+  per driver without changing the global URL normalization setting.
+  (GITHUB-1654)
+  [Shubham Padkonde - @Shubham-Padkonde]
+
 - [Azure Blobs] Fix SAS signatures for percent-encoded object paths by using
   the decoded path in the canonicalized resource while preserving the encoded
   path in the returned URL.

docs/storage/drivers/s3.rst

@@ -8,12 +8,29 @@ from Amazon Web Services.
 
     If you are upgrading from Libcloud v2.3.0 or older versions and are
     utilizing paths with duplicated slashes (e.g. ``/my-bucket//path/1.txt``)
-    or a root bucked named as ``/``, you will need to utilize
-    ``libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES`` variable which was
-    added in Libcloud v3.3.0 so you can access objects in those paths.
+    or a root bucket named as ``/``, enable the per-driver option described
+    below or use the global ``libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES``
+    variable added in Libcloud v3.3.0.
 
     For more information, please refer to the "Upgrade Notes".
 
+Object names containing repeated slashes
+----------------------------------------
+
+S3 treats slashes as part of an object's name. To preserve leading or repeated
+slashes when accessing these objects, enable the per-driver option::
+
+    driver = S3StorageDriver(key, secret, ex_allow_path_double_slashes=True)
+
+This applies to object requests, including uploads, downloads, and deletions.
+Without this option or the global ``ALLOW_PATH_DOUBLE_SLASHES`` setting, a name
+such as ``/path/to/object`` can be normalized to ``path/to/object``. S3 may then
+report a successful deletion even though the intended object still exists.
+
+The option defaults to ``False`` for compatibility. It does not alter other
+driver instances or the global setting. The global setting continues to apply
+when the per-driver option is not enabled.
+
 Multipart uploads
 -----------------
 

libcloud/storage/drivers/s3.py

@@ -250,6 +250,12 @@ class S3SignatureV4Connection(SignedAWSConnection, BaseS3Connection):
     service_name = "s3"
     version = API_VERSION
 
+    def morph_action_hook(self, action: str) -> str:
+        """Preserve object key slashes when enabled for this S3 driver."""
+        if getattr(self.driver, "ex_allow_path_double_slashes", False):
+            return self.request_path + action
+        return super().morph_action_hook(action)
+
     def __init__(
         self,
         user_id,
@@ -1243,8 +1249,17 @@ def __init__(
         port=None,
         region=None,
         token=None,
+        ex_allow_path_double_slashes: bool = False,
         **kwargs,
     ):
+        """
+        :param ex_allow_path_double_slashes: Preserve repeated slashes in object
+            request paths for this driver. When False, use the existing global
+            ``ALLOW_PATH_DOUBLE_SLASHES`` setting.
+        :type ex_allow_path_double_slashes: ``bool``
+        """
+        self.ex_allow_path_double_slashes = ex_allow_path_double_slashes
+
         # Here for backward compatibility for old and deprecated driver class
         # per region approach
         if hasattr(self, "region_name") and not region:

libcloud/test/storage/test_s3.py

@@ -1306,6 +1306,33 @@ def test_delete_object_success(self):
         result = self.driver.delete_object(obj=obj)
         self.assertTrue(result)
 
+    def test_delete_object_double_slashes(self) -> None:
+        for name in ("/object", "path//object", "//path///object"):
+            with self.subTest(name=name):
+                driver = S3StorageDriver(*self.driver_args, ex_allow_path_double_slashes=True)
+                container = Container(name="test-bucket", extra={}, driver=driver)
+                obj = Object(name, 0, None, {}, {}, container, driver)
+                with mock.patch.object(
+                    self.mock_response_klass,
+                    "_get_request",
+                    return_value=(httplib.NO_CONTENT, "", {}, "No Content"),
+                ) as request:
+                    self.assertTrue(driver.delete_object(obj))
+                self.assertEqual(request.call_args[0][0], "DELETE")
+                self.assertEqual(
+                    urlparse.urlparse(request.call_args[0][1]).path, "/test-bucket/" + name
+                )
+
+    def test_double_slashes_option_is_per_driver(self) -> None:
+        with mock.patch("libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES", False):
+            driver = S3StorageDriver(*self.driver_args, ex_allow_path_double_slashes=True)
+            other = S3StorageDriver(*self.driver_args)
+            path = "/test-bucket//object"
+            self.assertEqual(driver.connection.morph_action_hook(path), path)
+            self.assertEqual(other.connection.morph_action_hook(path), "/test-bucket/object")
+        with mock.patch("libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES", True):
+            self.assertEqual(other.connection.morph_action_hook(path), path)
+
     def test_region_keyword_argument(self):
         # Default region
         driver = S3StorageDriver(*self.driver_args)