文章/答案/技术大牛

发布

社区首页 >问答首页 >GCP云存储- Golang sdk2 -带有s3互操作性Cred的上传文件

问GCP云存储- Golang sdk2 -带有s3互操作性Cred的上传文件
EN

Stack Overflow用户

提问于 2022-09-14 13:11:38

回答 2查看 220关注 0票数 4

我正在尝试通过s3 go sdk aws-sdk-go-v2使用互操作性特征实现从云存储中的存储桶下载/上传文件到存储桶

下载工作如愿，但上传不起作用，有以下错误消息：SDK 2022/09/14 11:24:43 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.

由于我在下载和上传时都使用相同的access_key和secret_key，这似乎不是凭据问题。此外，hmac密钥后面的服务帐户具有角色。

在这里，代码：

main.go

package main

import (
    "context"
    "fmt"
    "os"
    "strings"

    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

var BUCKET_NAME = ""

func main() {

    //prepare gcp resolver
    gcpResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
        return aws.Endpoint{
            URL:               "https://storage.googleapis.com",
            SigningRegion:     "auto",
            Source:            aws.EndpointSourceCustom,
            HostnameImmutable: true,
        }, nil
    })
    //file with fornat : $accessKey:$secretKey
    file, _ := os.ReadFile("/home/bapt/creds/amz-keys-gcp-2")
    keys := strings.Split(string(file), ":")

    //init the config options
    optConfig := []func(*config.LoadOptions) error{
        config.WithRegion("auto"),
        config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(keys[0], strings.TrimRight(keys[1], "\n"), "")),
        config.WithClientLogMode(aws.LogRetries | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRequestEventMessage | aws.LogResponseEventMessage | aws.LogSigning),
        config.WithEndpointResolverWithOptions(gcpResolver),
    }

    //init config
    cfg, _ := config.LoadDefaultConfig(context.TODO(), optConfig...)

    //init service
    svc := s3.NewFromConfig(cfg)
    tempFile, _ := os.CreateTemp("/tmp", "test-gcp-*")
    defer tempFile.Close()
    downloader := manager.NewDownloader(svc)
    downloader.Download(context.TODO(),tempFile, &s3.GetObjectInput{
        Bucket: aws.String(BUCKET_NAME),
        Key:    aws.String("file-test.txt"),
    })
    //init uploader ( no multipart)
    uploader := manager.NewUploader(svc, func(u *manager.Uploader) {
        u.Concurrency = 1
        u.MaxUploadParts = 1
    })
    //upload
    _, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
        Bucket: aws.String(BUCKET_NAME),
        Key:    aws.String("file-test.txt"),
        Body:   strings.NewReader("HELLO"),
    })
    fmt.Println(err)
}

go.mod

module gcps3/v2

go 1.18

require (
    github.com/aws/aws-sdk-go-v2 v1.16.14
    github.com/aws/aws-sdk-go-v2/config v1.17.5
    github.com/aws/aws-sdk-go-v2/credentials v1.12.18
    github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.31
    github.com/aws/aws-sdk-go-v2/service/s3 v1.27.9
)

require (
    github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.7 // indirect
    github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.15 // indirect
    github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.21 // indirect
    github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.15 // indirect
    github.com/aws/aws-sdk-go-v2/internal/ini v1.3.22 // indirect
    github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.12 // indirect
    github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.8 // indirect
    github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.16 // indirect
    github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.15 // indirect
    github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.15 // indirect
    github.com/aws/aws-sdk-go-v2/service/sso v1.11.21 // indirect
    github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.3 // indirect
    github.com/aws/aws-sdk-go-v2/service/sts v1.16.17 // indirect
    github.com/aws/smithy-go v1.13.2 // indirect
    github.com/jmespath/go-jmespath v0.4.0 // indirect
)

在这里，PUT的调试跟踪(我的桶名被 bucket 替换，access_key替换为GOOG1ID )。

SDK 2022/09/14 14:52:37 DEBUG Request Signature:
---[ CANONICAL STRING  ]-----------------------------
PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z

accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD
---[ STRING TO SIGN ]--------------------------------
AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
bc09.....daf520
-----------------------------------------------------
SDK 2022/09/14 14:52:37 DEBUG Request
PUT /BUCKET/file-test.txt?x-id=PutObject HTTP/1.1
Host: storage.googleapis.com
User-Agent: aws-sdk-go-v2/1.16.14 os/linux lang/go/1.18.1 md/GOOS/linux md/GOARCH/amd64 api/s3/1.27.9 ft/s3-transfer
Content-Length: 5
Accept-Encoding: identity
Amz-Sdk-Invocation-Id: d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
Amz-Sdk-Request: attempt=1; max=3
Authorization: AWS4-HMAC-SHA256 Credential=GOOG1ID/20220914/auto/s3/aws4_request, SignedHeaders=accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date, Signature=c994....37d0
Content-Type: application/octet-stream
X-Amz-Content-Sha256: UNSIGNED-PAYLOAD
X-Amz-Date: 20220914T125237Z

HELLO
SDK 2022/09/14 14:52:37 DEBUG Response
HTTP/2.0 403 Forbidden
Content-Length: 883
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Content-Type: application/xml; charset=UTF-8
Date: Wed, 14 Sep 2022 12:52:38 GMT
Server: UploadServer
X-Guploader-Uploadid: ADPycdt0aCu6BTmzWQl2Ehc4q2sP8rtexDb4Keyn6cQL_GigREvc8T1CzX0HH-ZXgw_6XWLJPPYXufwRCr0Sl7uSsiIi0Q

<?xml version='1.0' encoding='UTF-8'?><Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.</Message><StringToSign>AWS4-HMAC-SHA256
20220914T125237Z
20220914/auto/s3/aws4_request
0a10....d63e</StringToSign><CanonicalRequest>PUT
/BUCKET/file-test.txt
x-id=PutObject
accept-encoding:identity,gzip(gfe)
amz-sdk-invocation-id:d6776820-e336-4bdf-afa3-0ca3b6d5b0a0
amz-sdk-request:attempt=1; max=3
content-length:5
content-type:application/octet-stream
host:storage.googleapis.com
x-amz-content-sha256:UNSIGNED-PAYLOAD
x-amz-date:20220914T125237Z

accept-encoding;amz-sdk-invocation-id;amz-sdk-request;content-length;content-type;host;x-amz-content-sha256;x-amz-date
UNSIGNED-PAYLOAD</CanonicalRequest></Error>
SDK 2022/09/14 14:52:37 DEBUG request failed with unretryable error https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.
operation error S3: PutObject, https response error StatusCode: 403, RequestID: , HostID: , api error SignatureDoesNotMatch: The request signature we calculated does not match the signature you provided. Check your Google secret key and signing method.

我用这个python脚本测试了上传，它可以工作(相同的cred)

boto3.set_stream_logger('', logging.DEBUG)
GCP_BUCKET = True
FILE_TO_BE_UPLOADED = '/tmp/toto'

if GCP_BUCKET:
    ACCESS_KEY = "ACESS"
    SECRET_KEY = "SECRET"
    bucket_name = "BUCKET"    
    region_name="auto"
    endpoint_url="https://storage.googleapis.com"
    
def makeS3Client():
    s3 = boto3.client("s3", 
                region_name=region_name,
                endpoint_url=endpoint_url,
                aws_access_key_id=ACCESS_KEY,
                aws_secret_access_key=SECRET_KEY,
                )

    return s3


def upload_file(s3,bucket_name,fname):
    """
    Uploads file to S3 bucket using S3 client object
    :return: None
    """
    object_name = os.path.basename(fname)
    file_name = os.path.abspath(fname)
    #file_name = os.path.join(pathlib.Path(__file__).parent.resolve(), fname)

    response = s3.upload_file(file_name, bucket_name, object_name)
    #print(response)  # prints None

我在https请求中看到的go和python之间唯一不同的地方是使用的signedHeaders。

但是带有s3 aws桶的go代码运行良好.

我错过了选择吗？

谢谢你的帮助。

amazon-s3

google-cloud-platform

google-cloud-storage

回答 2

Stack Overflow用户

发布于 2022-11-09 23:38:21

@h3y鸭子提到的问题是，接受编码似乎需要排除在v2库中的签名签名过程之外。由于签名依赖于SignedHeaders中提到的头，所以我们必须自己重新计算签名。

AWS配置对象公开用于所有服务的自定义http客户端的变量。我们可以通过定义自己的RoundTripper来使用它，它将请求的签名修改为不考虑接受编码的签名。

示例：

package main

import (
    "context"
    "fmt"
    "net/http"
    "net/http/httputil"
    "strings"
    "time"

    "github.com/aws/aws-sdk-go-v2/aws"
    v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

var BUCKET_NAME = "test"

type RecalculateV4Signature struct {
    next   http.RoundTripper
    signer *v4.Signer
    cfg    aws.Config
}

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error) {
    // store for later use
    val := req.Header.Get("Accept-Encoding")

    // delete the header so the header doesn't account for in the signature
    req.Header.Del("Accept-Encoding")

    // sign with the same date
    timeString := req.Header.Get("X-Amz-Date")
    timeDate, _ := time.Parse("20060102T150405Z", timeString)

    creds, _ := lt.cfg.Credentials.Retrieve(req.Context())
    err := lt.signer.SignHTTP(req.Context(), creds, req, v4.GetPayloadHash(req.Context()), "s3", lt.cfg.Region, timeDate)
    if err != nil {
        return nil, err
    }
    // Reset Accept-Encoding if desired
    req.Header.Set("Accept-Encoding", val)

    fmt.Println("AfterAdjustment")
    rrr, _ := httputil.DumpRequest(req, false)
    fmt.Println(string(rrr))

    // follows up the original round tripper
    return lt.next.RoundTrip(req)
}

func main() {

    //prepare gcp resolver
    gcpResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
        return aws.Endpoint{
            URL:               "https://storage.googleapis.com",
            SigningRegion:     "auto",
            Source:            aws.EndpointSourceCustom,
            HostnameImmutable: true,
        }, nil
    })
    //file with format : $accessKey:$secretKey

    //init the config options
    optConfig := []func(*config.LoadOptions) error{
        config.WithRegion("auto"),
        config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "session")),
        config.WithClientLogMode(aws.LogRetries | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRequestEventMessage | aws.LogResponseEventMessage | aws.LogSigning),
        config.WithEndpointResolverWithOptions(gcpResolver),
    }

    //init config
    cfg, _ := config.LoadDefaultConfig(context.TODO(), optConfig...)
    // Assign custom client with our own transport
    cfg.HTTPClient = &http.Client{Transport: &RecalculateV4Signature{http.DefaultTransport, v4.NewSigner(), cfg}}
    //init service
    svc := s3.NewFromConfig(cfg)
    uploader := manager.NewUploader(svc, func(u *manager.Uploader) {
        u.Concurrency = 1
        u.MaxUploadParts = 1
    })
    //upload
    _, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
        Bucket: aws.String(BUCKET_NAME),
        Key:    aws.String("file-test.txt"),
        Body:   strings.NewReader("HELLO"),
    })
    fmt.Println(err)
}

在理想情况下，客户端应该建立在aws的BuildableClient上，因为它们为它们的服务提供了合理的缺省值，我将由读者来决定。

票数 1

Stack Overflow用户

发布于 2022-11-09 16:28:05

根据https://github.com/aws/aws-sdk-go-v2/issues/1816的说法，v2 SDK不支持GCP。

从叹息的标题中删除Accept-Encoding为我解决了这个问题，但我不确定这是否是一个明智的做法。此外，除了修改库的源代码(将其添加到IgnoredHeaders全局变量)之外，我没有找到从签名头列表中删除此标头的方法(可能有一个中间件支持此方法)。

票数 0

页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持

原文链接：

https://stackoverflow.com/questions/73717477

复制

相似问题

问GCP云存储- Golang sdk2 -带有s3互操作性Cred的上传文件
EN

回答 2

Stack Overflow用户

Stack Overflow用户

社区

活动

圈层

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问GCP云存储- Golang sdk2 -带有s3互操作性Cred的上传文件EN

回答 2

Stack Overflow用户

Stack Overflow用户

社区

活动

圈层

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问GCP云存储- Golang sdk2 -带有s3互操作性Cred的上传文件
EN