Add a basic support for parsing urls from virtual private cloud (#49)

* Add a basic support for parsing urls from virtual private cloud

* Improve comment

* v1.6.2

* PR feedback

* 1.6 compat

Author: Drvi

Project.toml

@@ -1,7 +1,7 @@
 name = "CloudStore"
 uuid = "3365d9ee-d53b-4a56-812d-5344d5b716d7"
 authors = ["quinnj <[email protected]>"]
-version = "1.6.1"
+version = "1.6.2"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/parse.jl

@@ -18,6 +18,7 @@ const AWS_REGIONS = Set{String}([
     "ap-southeast-2",
     "ap-northeast-1",
     "ca-central-1",
+    "ca-west-1",
     "eu-central-1",
     "eu-west-1",
     "eu-west-2",
@@ -26,6 +27,7 @@ const AWS_REGIONS = Set{String}([
     "eu-south-2",
     "eu-north-1",
     "eu-central-2",
+    "il-central-1",
     "me-south-1",
     "me-central-1",
     "sa-east-1",
@@ -136,6 +138,12 @@ function parseAWSBucketRegionKey(url; parseLocal::Bool=false)
     # https://bucket-name.s3.amazonaws.com
     m = match(r"^https://(?<bucket>[^\.]+)\.s3(?<accelerate>-accelerate)?(?:\.(?<region>[^\.]+))?\.amazonaws\.com(?:/(?<key>.+))?$"i, url)
     m !== nothing && return _validate_aws(true, !isnothing(m[:accelerate]), nothing, m[:bucket], m[:region], m[:key])
+
+    ### See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html
+    # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name/key-name
+    # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name
+    m = match(r"^https://bucket\.vpce[^\.]+\.s3\.(?<region>[^\.]+)\.vpce\.amazonaws\.com/(?<bucket>[^/]+)(?:/(?<key>.+))?$"i, url)
+    m !== nothing && return _validate_aws(true, false, nothing, m[:bucket], m[:region], m[:key])
     # https://s3.region-code.amazonaws.com/bucket-name/key-name
     # https://s3.region-code.amazonaws.com/bucket-name
     m = match(r"^https://s3(?:\.(?<region>[^\.]+))?\.amazonaws\.com/(?<bucket>[^/]+)(?:/(?<key>.+))?$"i, url)

test/runtests.jl

@@ -531,6 +531,9 @@ end
         ("S3://bucket-name", (true, false, nothing, "bucket-name", "", "")),
         ("HTtp://127.0.0.1:27181/bucket-name/key-name", (true, false, "HTtp://127.0.0.1:27181", "bucket-name", "", "key-name")),
         ("htTP://127.0.0.1:27181/bucket-name", (true, false, "htTP://127.0.0.1:27181", "bucket-name", "", "")),
+
+        ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name", (true, false, nothing, "bucket-name", "us-west-2", "")),
+        ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name", (true, false, nothing, "bucket-name", "us-west-2", "key-name")),
     ]
     for (url, parts) in s3
         ok, accelerate, host, bucket, reg, key = CloudStore.parseAWSBucketRegionKey(url; parseLocal=true)
@@ -587,6 +590,12 @@ end
         "httP://S3.AmAzonAws.com/bucket-name",
         "httP://bucket-name/key-name",
         "httP://bucket-name",
+
+        "http://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name",
+        "http://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name",
+        "https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.XvpceX.amazonaws.com/bucket-name",
+        "https://bucket.Xvpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name",
+        "https://XbucketX.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name",
     ]
     for url in invalid_s3
         ok, accelerate, host, bucket, reg, key = CloudStore.parseAWSBucketRegionKey(url; parseLocal=true)
@@ -918,6 +927,10 @@ end
         @test CloudStore.validate_bucket_name("a.b-c1", false) == "a.b-c1"
         @test CloudStore.validate_bucket_name("a"^63, false) == "a"^63
         @test CloudStore.validate_bucket_name("a"^3, false) == "a"^3
+
+        @test_throws ArgumentError("Validation failed for `region` \"xx-xxxx-x\"") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.xx-xxxx-x.vpce.amazonaws.com/bucket-name")
+        @test_throws ArgumentError("Validation failed for `bucket` name \"bn\": Bucket names must be between 3 (min) and 63 (max) characters long.") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bn")
+        @test_throws ArgumentError("Validation failed for `key` \"key-n$("a" ^ 1024)me\": The key name must be shorter than 1025 bytes.") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-n$("a" ^ 1024)me")
     end
 
     @testset "validate_container_name" begin