From ce971a32b81312ec4bb574e37cab0c89b9432558 Mon Sep 17 00:00:00 2001 From: Ambrose Chua Date: Sat, 22 May 2021 23:53:00 +0800 Subject: [PATCH] Move uploader to subpath --- assets.go | 2 +- handlers-s3.go | 248 ++++++++++++++++++++++++++++++++ handlers.go | 247 ++----------------------------- main.go | 17 ++- s3.go | 2 +- web/src/main.js | 2 +- web/{index.html => upload.tmpl} | 0 7 files changed, 269 insertions(+), 249 deletions(-) create mode 100644 handlers-s3.go rename web/{index.html => upload.tmpl} (100%) diff --git a/assets.go b/assets.go index d3a0b76..d553b7f 100644 --- a/assets.go +++ b/assets.go @@ -4,5 +4,5 @@ import ( "embed" ) -//go:embed web/index.html web/assets/* +//go:embed web/*.tmpl web/assets/* var assets embed.FS diff --git a/handlers-s3.go b/handlers-s3.go new file mode 100644 index 0000000..6a87fd2 --- /dev/null +++ b/handlers-s3.go @@ -0,0 +1,248 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "strconv" + + "github.com/gorilla/mux" +) + +/* createMultipartUpload */ + +type createMultipartUploadReq struct { + Filename string `json:"filename"` + Type string `json:"type"` + Metadata createMultipartUploadReqMetadata `json:"metadata"` +} + +func (r createMultipartUploadReq) validate() error { + if r.Filename == "" { + return errors.New("invalid filename") + } else if r.Type == "" { + return errors.New("invalid content type") + } + return nil +} + +type createMultipartUploadReqMetadata struct { + Name string `json:"name"` + Type string `json:"type"` +} + +type createMultipartUploadRes struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` +} + +func handleCreateMultipartUpload(w http.ResponseWriter, req *http.Request) { + r := createMultipartUploadReq{} + decoder := json.NewDecoder(req.Body) + if err := decoder.Decode(&r); err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) + return + } + + if err := r.validate(); err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) + return + } + + cred := credential{ + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + Region: os.Getenv("MINIO_REGION_NAME"), + Endpoint: os.Getenv("MINIO_ENDPOINT"), + Prefix: os.Getenv("PREFIX"), + } + + // Derive the object key + key := cred.Prefix + r.Filename + + result, err := initiateMultipartUpload(key, cred) + if err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) + return + } + + encoder := json.NewEncoder(w) + encoder.Encode(createMultipartUploadRes{ + Key: key, + UploadID: result.UploadID, + }) +} + +/* getUploadedParts */ + +type getUploadedPartsRes []part + +func handleGetUploadedParts(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + uploadID := vars["uploadID"] + key := req.URL.Query().Get("key") + + if uploadID == "" || key == "" { + errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) + return + } + + cred := credential{ + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + Region: os.Getenv("MINIO_REGION_NAME"), + Endpoint: os.Getenv("MINIO_ENDPOINT"), + Prefix: os.Getenv("PREFIX"), + } + + parts := make(getUploadedPartsRes, 0, 0) + var nextPartNumberMarker uint32 + for { + page, err := listParts(key, uploadID, cred, nextPartNumberMarker) + if err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) + return + } + + parts = append(parts, page.Parts...) + nextPartNumberMarker = page.NextPartNumberMarker + + if !page.IsTruncated { + break + } + } + + encoder := json.NewEncoder(w) + encoder.Encode(getUploadedPartsRes(parts)) +} + +/* signPartUpload */ + +type signPartUploadRes struct { + URL string `json:"url"` +} + +func handleSignPartUpload(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + uploadID := vars["uploadID"] + key := req.URL.Query().Get("key") + partNumber, err := strconv.ParseUint(vars["uploadPart"], 10, 16) + + if uploadID == "" || key == "" { + errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) + return + } + if partNumber < 1 || partNumber > 10000 || err != nil { + errorResponse(w, req, fmt.Errorf("%w: invalid part number", errBadRequest)) + return + } + + cred := credential{ + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + Region: os.Getenv("MINIO_REGION_NAME"), + Endpoint: os.Getenv("MINIO_ENDPOINT"), + Prefix: os.Getenv("PREFIX"), + } + + params := make(url.Values) + params.Add("partNumber", strconv.FormatUint(partNumber, 10)) + params.Add("uploadId", uploadID) + unsignedReq, err := http.NewRequest(http.MethodPut, cred.Endpoint+"/"+key+"?"+params.Encode(), nil) + if err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) + return + } + + signedReq := preSign(unsignedReq, cred) + + encoder := json.NewEncoder(w) + encoder.Encode(signPartUploadRes{ + URL: signedReq.URL.String(), + }) +} + +/* completeMultipartUpload */ + +type completeMultipartUploadReq struct { + Parts []completePart `json:"parts"` +} + +func (r completeMultipartUploadReq) validate() error { + for _, part := range r.Parts { + if err := part.validate(); err != nil { + return err + } + } + return nil +} + +func handleCompleteMultipartUpload(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + uploadID := vars["uploadID"] + key := req.URL.Query().Get("key") + + if uploadID == "" || key == "" { + errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) + return + } + + r := completeMultipartUploadReq{} + decoder := json.NewDecoder(req.Body) + if err := decoder.Decode(&r); err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) + return + } + + if err := r.validate(); err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) + return + } + + cred := credential{ + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + Region: os.Getenv("MINIO_REGION_NAME"), + Endpoint: os.Getenv("MINIO_ENDPOINT"), + Prefix: os.Getenv("PREFIX"), + } + + result, err := completeMultipartUpload(key, uploadID, r.Parts, cred) + if err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) + return + } + + encoder := json.NewEncoder(w) + encoder.Encode(result) +} + +/* abortMultipartUpload */ + +func handleAbortMultipartUpload(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + uploadID := vars["uploadID"] + key := req.URL.Query().Get("key") + + if uploadID == "" || key == "" { + errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) + return + } + + cred := credential{ + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + Region: os.Getenv("MINIO_REGION_NAME"), + Endpoint: os.Getenv("MINIO_ENDPOINT"), + Prefix: os.Getenv("PREFIX"), + } + + err := abortMultipartUpload(key, uploadID, cred) + if err != nil { + errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) + return + } +} diff --git a/handlers.go b/handlers.go index 4cf090e..5f96d35 100644 --- a/handlers.go +++ b/handlers.go @@ -1,15 +1,9 @@ package main import ( - "encoding/json" - "errors" - "fmt" + "html/template" "net/http" - "net/url" "os" - "strconv" - - "github.com/gorilla/mux" ) var globalStore store @@ -22,237 +16,12 @@ func setupHandlers() { } } -/* createMultipartUpload */ +/* templates */ -type createMultipartUploadReq struct { - Filename string `json:"filename"` - Type string `json:"type"` - Metadata createMultipartUploadReqMetadata `json:"metadata"` -} - -func (r createMultipartUploadReq) validate() error { - if r.Filename == "" { - return errors.New("invalid filename") - } else if r.Type == "" { - return errors.New("invalid content type") - } - return nil -} - -type createMultipartUploadReqMetadata struct { - Name string `json:"name"` - Type string `json:"type"` -} - -type createMultipartUploadRes struct { - Key string `json:"key"` - UploadID string `json:"uploadId"` -} - -func handleCreateMultipartUpload(w http.ResponseWriter, req *http.Request) { - r := createMultipartUploadReq{} - decoder := json.NewDecoder(req.Body) - if err := decoder.Decode(&r); err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) - return - } - - if err := r.validate(); err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) - return - } - - cred := credential{ - AccessKey: os.Getenv("MINIO_ACCESS_KEY"), - SecretKey: os.Getenv("MINIO_SECRET_KEY"), - Region: os.Getenv("MINIO_REGION_NAME"), - Endpoint: os.Getenv("MINIO_ENDPOINT"), - Prefix: os.Getenv("PREFIX"), - } - - // Derive the object key - key := cred.Prefix + r.Filename - - result, err := initiateMultipartUpload(key, cred) - if err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) - return - } - - encoder := json.NewEncoder(w) - encoder.Encode(createMultipartUploadRes{ - Key: key, - UploadID: result.UploadID, - }) -} - -/* getUploadedParts */ - -type getUploadedPartsRes []part - -func handleGetUploadedParts(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - uploadID := vars["id"] - key := req.URL.Query().Get("key") - - if uploadID == "" || key == "" { - errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) - return - } - - cred := credential{ - AccessKey: os.Getenv("MINIO_ACCESS_KEY"), - SecretKey: os.Getenv("MINIO_SECRET_KEY"), - Region: os.Getenv("MINIO_REGION_NAME"), - Endpoint: os.Getenv("MINIO_ENDPOINT"), - Prefix: os.Getenv("PREFIX"), - } - - parts := make(getUploadedPartsRes, 0, 0) - var nextPartNumberMarker uint32 - for { - page, err := listParts(key, uploadID, cred, nextPartNumberMarker) - if err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) - return - } - - parts = append(parts, page.Parts...) - nextPartNumberMarker = page.NextPartNumberMarker - - if !page.IsTruncated { - break - } - } - - encoder := json.NewEncoder(w) - encoder.Encode(getUploadedPartsRes(parts)) -} - -/* signPartUpload */ - -type signPartUploadRes struct { - URL string `json:"url"` -} - -func handleSignPartUpload(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - uploadID := vars["id"] - key := req.URL.Query().Get("key") - partNumber, err := strconv.ParseUint(vars["part"], 10, 16) - - if uploadID == "" || key == "" { - errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) - return - } - if partNumber < 1 || partNumber > 10000 || err != nil { - errorResponse(w, req, fmt.Errorf("%w: invalid part number", errBadRequest)) - return - } - - cred := credential{ - AccessKey: os.Getenv("MINIO_ACCESS_KEY"), - SecretKey: os.Getenv("MINIO_SECRET_KEY"), - Region: os.Getenv("MINIO_REGION_NAME"), - Endpoint: os.Getenv("MINIO_ENDPOINT"), - Prefix: os.Getenv("PREFIX"), - } - - params := make(url.Values) - params.Add("partNumber", strconv.FormatUint(partNumber, 10)) - params.Add("uploadId", uploadID) - unsignedReq, err := http.NewRequest(http.MethodPut, cred.Endpoint+"/"+key+"?"+params.Encode(), nil) - if err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) - return - } - - signedReq := preSign(unsignedReq, cred) - - encoder := json.NewEncoder(w) - encoder.Encode(signPartUploadRes{ - URL: signedReq.URL.String(), - }) -} - -/* completeMultipartUpload */ - -type completeMultipartUploadReq struct { - Parts []completePart `json:"parts"` -} - -func (r completeMultipartUploadReq) validate() error { - for _, part := range r.Parts { - if err := part.validate(); err != nil { - return err - } - } - return nil -} - -func handleCompleteMultipartUpload(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - uploadID := vars["id"] - key := req.URL.Query().Get("key") - - if uploadID == "" || key == "" { - errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) - return - } - - r := completeMultipartUploadReq{} - decoder := json.NewDecoder(req.Body) - if err := decoder.Decode(&r); err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) - return - } - - if err := r.validate(); err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errBadRequest, err)) - return - } - - cred := credential{ - AccessKey: os.Getenv("MINIO_ACCESS_KEY"), - SecretKey: os.Getenv("MINIO_SECRET_KEY"), - Region: os.Getenv("MINIO_REGION_NAME"), - Endpoint: os.Getenv("MINIO_ENDPOINT"), - Prefix: os.Getenv("PREFIX"), - } - - result, err := completeMultipartUpload(key, uploadID, r.Parts, cred) - if err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) - return - } - - encoder := json.NewEncoder(w) - encoder.Encode(result) -} - -/* abortMultipartUpload */ - -func handleAbortMultipartUpload(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - uploadID := vars["id"] - key := req.URL.Query().Get("key") - - if uploadID == "" || key == "" { - errorResponse(w, req, fmt.Errorf("%w", errBadRequest)) - return - } - - cred := credential{ - AccessKey: os.Getenv("MINIO_ACCESS_KEY"), - SecretKey: os.Getenv("MINIO_SECRET_KEY"), - Region: os.Getenv("MINIO_REGION_NAME"), - Endpoint: os.Getenv("MINIO_ENDPOINT"), - Prefix: os.Getenv("PREFIX"), - } - - err := abortMultipartUpload(key, uploadID, cred) - if err != nil { - errorResponse(w, req, fmt.Errorf("%w: %s", errInternalServerError, err)) - return - } +var tmpl = template.Must(template.ParseFS(assets, "web/*.tmpl")) + +/* upload template */ + +func handleUpload(w http.ResponseWriter, req *http.Request) { + tmpl.ExecuteTemplate(w, "upload.tmpl", nil) } diff --git a/main.go b/main.go index fc473f4..017a987 100644 --- a/main.go +++ b/main.go @@ -24,14 +24,17 @@ func main() { setupHandlers() router := mux.NewRouter() - multipartRouter := router.PathPrefix("/s3/multipart").Subrouter() - router.PathPrefix("/").Handler(http.FileServer(http.FS(assetsWeb))) + uploadRouter := router.PathPrefix("/{id}").Subrouter() + router.PathPrefix("/assets").Handler(http.FileServer(http.FS(assetsWeb))) - multipartRouter.HandleFunc("", handleCreateMultipartUpload).Methods(http.MethodPost) - multipartRouter.HandleFunc("/{id}", handleGetUploadedParts).Methods(http.MethodGet) - multipartRouter.HandleFunc("/{id}/{part}", handleSignPartUpload).Methods(http.MethodGet) - multipartRouter.HandleFunc("/{id}/complete", handleCompleteMultipartUpload).Methods(http.MethodPost) - multipartRouter.HandleFunc("/{id}", handleAbortMultipartUpload).Methods(http.MethodDelete) + uploadRouter.Path("").HandlerFunc(handleUpload) + s3Router := uploadRouter.PathPrefix("/s3/multipart").Subrouter() + + s3Router.Methods(http.MethodPost).Path("").HandlerFunc(handleCreateMultipartUpload) + s3Router.Methods(http.MethodGet).Path("/{uploadID}").HandlerFunc(handleGetUploadedParts) + s3Router.Methods(http.MethodGet).Path("/{uploadID}/{uploadPart}").HandlerFunc(handleSignPartUpload) + s3Router.Methods(http.MethodPost).Path("/{uploadID}/complete").HandlerFunc(handleCompleteMultipartUpload) + s3Router.Methods(http.MethodDelete).Path("/{uploadID}").HandlerFunc(handleAbortMultipartUpload) server := &http.Server{ Handler: router, diff --git a/s3.go b/s3.go index b2b93ca..52da727 100644 --- a/s3.go +++ b/s3.go @@ -273,7 +273,7 @@ func abortMultipartUpload(key, uploadID string, cred credential) error { return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusNoContent { body, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("endpoint request failed: %d: %s", resp.StatusCode, body) } diff --git a/web/src/main.js b/web/src/main.js index 5045bbf..5850950 100644 --- a/web/src/main.js +++ b/web/src/main.js @@ -22,7 +22,7 @@ uppy.use(StatusBar, { }); uppy.use(AwsS3Multipart, { limit: 3, - companionUrl: '.', + companionUrl: window.location.pathname, }); const log = new Log('#log-area'); diff --git a/web/index.html b/web/upload.tmpl similarity index 100% rename from web/index.html rename to web/upload.tmpl