From 6070c648276c9bb4bfc160ebf3ad6acd4f0dae37 Mon Sep 17 00:00:00 2001 From: Ambrose Chua Date: Sun, 31 May 2020 18:24:35 +0800 Subject: [PATCH] Implement unauthenticated access control --- .gitignore | 3 +- README.md | 29 +-- cmd/control/control.go | 109 ++++++----- .../thumbnail.go => preview/preview.go} | 2 +- cmd/proxy/proxy.go | 27 ++- internal/s3utils/hostpath.go | 8 + pkg/bucket/client.go | 7 +- pkg/bucket/photo.go | 4 + pkg/bucket/signer.go | 176 ++++++++++++++++++ 9 files changed, 298 insertions(+), 67 deletions(-) rename cmd/{thumbnail/thumbnail.go => preview/preview.go} (98%) create mode 100644 internal/s3utils/hostpath.go create mode 100644 pkg/bucket/signer.go diff --git a/.gitignore b/.gitignore index 27392dd..95f53d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ cmd/admin/admin cmd/control/control cmd/web/web -cmd/thumbnail/thumbnail -cmd/indexer/indexer +cmd/preview/preview cmd/proxy/proxy env diff --git a/README.md b/README.md index 4d24afb..4de087d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,14 @@ A photo bucket management suite. +There are two modes of operation: +- Domain + - Buckets are exactly equal to their domain names + - `unset MINIO_DOMAIN` +- Subdomain + - Buckets are named after subdomains + - `export MINIO_DOMAIN=your.domain` + ## `admin` Create new buckets. Standalone tool. @@ -64,27 +72,20 @@ Generates the web interface for a photo bucket. Also updates the shared asset bu Regenerate and upload `index.html` and `manage/index.html` to bucket. -## `indexer` +## `preview` -Pointed to by a reverse proxy to handle the following paths on all buckets: - -- `/` -- `/manage/` - -#### `GET /*` - -A proxy for all buckets, treats the URL as a directory and serves up directory + `index.html`. - -## `thumbnail` - -Generate thumbnails from photo buckets. Registers webhooks. +Generate previews from photo buckets. Registers webhooks. ### Operations #### `POST /webhook` #### `POST /update?bucket=BUCKET&photo=OBJECT` -1. Perform thumbnail generation using libvips (maybe limit?) +1. Perform preview generation using libvips (maybe limit?) 2. Block until done +## `proxy` + +Reverse proxies buckets to the minio endpoint, as a substitute for the AWS S3 website hosting features. Serves up `index.html` when URLs end in a slash. + diff --git a/cmd/control/control.go b/cmd/control/control.go index c0d5944..44b17e7 100644 --- a/cmd/control/control.go +++ b/cmd/control/control.go @@ -3,72 +3,95 @@ package main import ( "fmt" "net/http" - "net/url" - "os" + "strconv" "time" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" + "git.makerforce.io/photos/photos/internal/httphelpers" + lib "git.makerforce.io/photos/photos/pkg/bucket" ) -var accessKey string -var secretKey string +var signer *lib.Signer func main() { - // Read configuration - accessKey = os.Getenv("MINIO_ACCESS_KEY") - secretKey = os.Getenv("MINIO_SECRET_KEY") + // Setup bucket signer + var err error + signer, err = lib.NewSignerFromEnv() server := &http.Server{ Addr: ":8000", ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, } - http.HandleFunc("/list", hello) - http.HandleFunc("/write", hello2) - err := server.ListenAndServe() + http.HandleFunc("/list", list) + http.HandleFunc("/read", read) + http.HandleFunc("/write", write) + err = server.ListenAndServe() if err != nil { panic(err) } } -func hello(w http.ResponseWriter, req *http.Request) { - // Based upon https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L873 - expires := 60 * time.Second - reqValues := make(url.Values) - reqValues.Set("list-type", "2") - reqValues.Set("metadata", "true") - reqValues.Set("encoding-type", "url") - reqValues.Set("prefix", "") - reqValues.Set("delimiter", "") - reqValues.Set("max-keys", fmt.Sprintf("%d", 500)) - reqValues.Set("start-after", "") - // Change path accordingly https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L911 - // Request URL MUST not have a port - reqString := "https://rock.six.six.six.six.six.eurica.eu.org/?" + s3utils.QueryEncode(reqValues) - req, err := http.NewRequest("GET", reqString, nil) - if err != nil { - panic(err) +func list(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) + httphelpers.ErrorResponse(w, err) + return } - signedReq := signer.PreSignV4(*req, accessKey, secretKey, "", "sgp1", int64(expires/time.Second)) - w.Header().Add("Location", signedReq.URL.String()) + bucket := lib.Bucket(req.FormValue("bucket")) + startAfter := req.FormValue("start-after") + maxKeysString := req.FormValue("max-keys") + if maxKeysString == "" { + maxKeysString = "4000" + } + maxKeys, err := strconv.Atoi(maxKeysString) // Let minio handle error + + url, err := signer.GetBucketPhotos(bucket, startAfter, maxKeys) + if err != nil { + httphelpers.ErrorResponse(w, err) + return + } + + w.Header().Add("Location", url) w.WriteHeader(http.StatusTemporaryRedirect) } -func hello2(w http.ResponseWriter, req *http.Request) { - // Based upon https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L873 - expires := 60 * time.Second - reqValues := make(url.Values) - // Change path accordingly https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L911 - // Request URL MUST not have a port - reqString := "https://rock.six.six.six.six.six.eurica.eu.org/rock.txt?" + s3utils.QueryEncode(reqValues) - req, err := http.NewRequest("PUT", reqString, nil) - if err != nil { - panic(err) +func read(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet { + err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) + httphelpers.ErrorResponse(w, err) + return } - signedReq := signer.PreSignV4(*req, accessKey, secretKey, "", "sgp1", int64(expires/time.Second)) - w.Header().Add("Location", signedReq.URL.String()) + bucket := lib.Bucket(req.FormValue("bucket")) + photo := lib.Photo(req.FormValue("photo")) + + url, err := signer.GetPhoto(bucket, photo) + if err != nil { + httphelpers.ErrorResponse(w, err) + return + } + + w.Header().Add("Location", url) + w.WriteHeader(http.StatusTemporaryRedirect) +} + +func write(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPut { + err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) + httphelpers.ErrorResponse(w, err) + return + } + + bucket := lib.Bucket(req.FormValue("bucket")) + photo := lib.Photo(req.FormValue("photo")) + + url, err := signer.PutPhoto(bucket, photo) + if err != nil { + httphelpers.ErrorResponse(w, err) + return + } + + w.Header().Add("Location", url) w.WriteHeader(http.StatusTemporaryRedirect) } diff --git a/cmd/thumbnail/thumbnail.go b/cmd/preview/preview.go similarity index 98% rename from cmd/thumbnail/thumbnail.go rename to cmd/preview/preview.go index 548a3cf..a61a567 100644 --- a/cmd/thumbnail/thumbnail.go +++ b/cmd/preview/preview.go @@ -7,7 +7,7 @@ import ( "sync" "time" - httphelpers "git.makerforce.io/photos/photos/internal/httphelpers" + "git.makerforce.io/photos/photos/internal/httphelpers" lib "git.makerforce.io/photos/photos/pkg/bucket" "github.com/davidbyttow/govips/pkg/vips" ) diff --git a/cmd/proxy/proxy.go b/cmd/proxy/proxy.go index 9dc0f6e..3f1a754 100644 --- a/cmd/proxy/proxy.go +++ b/cmd/proxy/proxy.go @@ -10,17 +10,20 @@ import ( "strings" "time" + internal_s3utils "git.makerforce.io/photos/photos/internal/s3utils" "github.com/miekg/dns" ) var endpoint string var endpointSecure bool +var domain string var behindProxy bool func main() { // Read configuration endpoint = os.Getenv("MINIO_ENDPOINT") endpointSecure = os.Getenv("MINIO_ENDPOINT_SECURE") == "true" + domain = os.Getenv("MINIO_DOMAIN") behindProxy = os.Getenv("BEHIND_PROXY") == "true" server := &http.Server{ @@ -53,7 +56,19 @@ func director(req *http.Request) { req.URL.Scheme = "https" } req.URL.Host = endpoint - req.URL.Path = mapPath(req.URL.Path, host) + if len(domain) < 1 { + // If we are using domains, rewrite the host into path + req.URL.Path = internal_s3utils.PathFromHost(mapPath(*req), host) + } else { + // If we are using subdomains, set the host header + req.URL.Path = mapPath(*req) + //req.Header.Set("Host", host) + } + + // Prevent MINIO from issuing redirects + userAgent := req.Header.Get("User-Agent") + userAgent = strings.ReplaceAll(userAgent, "Mozilla", "M-o-z-i-l-l-a") + req.Header.Set("User-Agent", userAgent) if !behindProxy { // Clear existing unsafe headers @@ -66,12 +81,16 @@ func director(req *http.Request) { req.Header.Set("X-Forwarded-Port", port) } - log.Println(req.URL) + log.Println(req.Header.Get("Host"), req.URL) } -func mapPath(path, host string) string { +func mapPath(req http.Request) string { + path := req.URL.Path + if req.FormValue("list-type") == "2" { + return path + } if strings.HasSuffix(path, "/") { path += "index.html" } - return "/" + host + path + return path } diff --git a/internal/s3utils/hostpath.go b/internal/s3utils/hostpath.go new file mode 100644 index 0000000..add08e5 --- /dev/null +++ b/internal/s3utils/hostpath.go @@ -0,0 +1,8 @@ +package s3utils + +func PathFromHost(path, host string) string { + if len(host) > 0 { + return "/" + host + path + } + return path +} diff --git a/pkg/bucket/client.go b/pkg/bucket/client.go index 734688a..2ad4318 100644 --- a/pkg/bucket/client.go +++ b/pkg/bucket/client.go @@ -13,8 +13,8 @@ type Client struct { *minio.Client } -func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Client, error) { - m, err := minio.New(endpoint, accessKey, secretKey, endpointSecure) +func NewClient(endpoint, accessKey, secretKey, regionName string, endpointSecure bool) (*Client, error) { + m, err := minio.NewWithRegion(endpoint, accessKey, secretKey, endpointSecure, regionName) if err != nil { return nil, err } @@ -28,9 +28,10 @@ func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Cli func NewClientFromEnv() (*Client, error) { accessKey := os.Getenv("MINIO_ACCESS_KEY") secretKey := os.Getenv("MINIO_SECRET_KEY") + regionName := os.Getenv("MINIO_REGION_NAME") endpoint := os.Getenv("MINIO_ENDPOINT") endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true" - return NewClient(endpoint, accessKey, secretKey, endpointSecure) + return NewClient(endpoint, accessKey, secretKey, regionName, endpointSecure) } func (c *Client) GetBucketMetadata(b Bucket) (BucketMetadata, error) { diff --git a/pkg/bucket/photo.go b/pkg/bucket/photo.go index cb5c4eb..a5c7bdb 100644 --- a/pkg/bucket/photo.go +++ b/pkg/bucket/photo.go @@ -22,3 +22,7 @@ func (p Photo) Validate() error { func (p Photo) String() string { return string(p) } + +func (p Photo) Path() string { + return "/" + string(p) +} diff --git a/pkg/bucket/signer.go b/pkg/bucket/signer.go new file mode 100644 index 0000000..7be4c77 --- /dev/null +++ b/pkg/bucket/signer.go @@ -0,0 +1,176 @@ +package bucket + +import ( + "errors" + "net" + "net/http" + "net/url" + "os" + "strconv" + "time" + + internal_s3utils "git.makerforce.io/photos/photos/internal/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/signer" +) + +type Signer struct { + accessKey string + secretKey string + regionName string + bucketSecure bool + expirations Expirations +} + +type Expirations struct { + // Expiration time for list and read in time.Duration + Read time.Duration + // Expiration time for write in time.Duration + Write time.Duration +} + +var ErrorExpirationTooLow = errors.New("expiration time too low") + +func NewSigner(accessKey, secretKey, regionName string, bucketSecure bool, expirations Expirations) (*Signer, error) { + if expirations.Read == 0 { + expirations.Read = 30 * time.Minute + } + if expirations.Write == 0 { + expirations.Write = 5 * time.Minute + } + if expirations.Read < time.Second || expirations.Write < time.Second { + return nil, ErrorExpirationTooLow + } + signer := &Signer{ + accessKey: accessKey, + secretKey: secretKey, + regionName: regionName, + bucketSecure: bucketSecure, + expirations: expirations, + } + return signer, nil +} + +func NewSignerFromEnv() (*Signer, error) { + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + regionName := os.Getenv("MINIO_REGION_NAME") + bucketSecure := os.Getenv("BUCKET_SECURE") == "true" + expirationRead, _ := strconv.Atoi(os.Getenv("EXPIRATION_READ")) + expirationWrite, _ := strconv.Atoi(os.Getenv("EXPIRATION_WRITE")) + expirations := Expirations{Read: time.Duration(expirationRead), Write: time.Duration(expirationWrite)} + return NewSigner(accessKey, secretKey, regionName, bucketSecure, expirations) +} + +func (s *Signer) baseBucket(b Bucket) url.URL { + url := url.URL{ + Scheme: "http", + Host: b.String(), + Path: "", + } + if s.bucketSecure { + url.Scheme = "https" + } + return url +} + +func (s *Signer) preSignV4(req http.Request) http.Request { + domain := os.Getenv("MINIO_DOMAIN") + + // Validate host + host, _, err := net.SplitHostPort(req.Host) + if err != nil { + host = req.Host + } + originalPath := req.URL.Path + + if len(domain) < 1 { + // If we are using domains, rewrite the host into path + req.URL.Path = internal_s3utils.PathFromHost(req.URL.Path, host) + } + // If we are using subdomains, sign the direct URL + signedReq := signer.PreSignV4( + req, + s.accessKey, s.secretKey, "", + "sgp1", + int64(s.expirations.Read/time.Second), + ) + signedReq.URL.Path = originalPath + return *signedReq +} + +func listBucketParams(prefix, startAfter string, maxKeys int) url.Values { + params := make(url.Values) + params.Set("list-type", "2") + params.Set("metadata", "true") + params.Set("encoding-type", "url") + params.Set("prefix", prefix) + params.Set("delimiter", "") + params.Set("max-keys", strconv.Itoa(maxKeys)) + params.Set("start-after", startAfter) + return params +} + +func (s *Signer) GetBucketPhotos(b Bucket, startAfter string, maxKeys int) (string, error) { + err := b.Validate() + if err != nil { + return "", err + } + + url := s.baseBucket(b) + url.Path += "/" + params := listBucketParams(photoPrefix, startAfter, maxKeys) + url.RawQuery = s3utils.QueryEncode(params) + + req, err := http.NewRequest("GET", url.String(), nil) + if err != nil { + return "", err + } + + signedReq := s.preSignV4(*req) + return signedReq.URL.String(), nil +} + +func (s *Signer) GetPhoto(b Bucket, p Photo) (string, error) { + err := b.Validate() + if err != nil { + return "", err + } + err = p.Validate() + if err != nil { + return "", err + } + + url := s.baseBucket(b) + url.Path += p.Path() + + req, err := http.NewRequest("GET", url.String(), nil) + if err != nil { + return "", err + } + + signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Read/time.Second)) + return signedReq.URL.String(), nil +} + +func (s *Signer) PutPhoto(b Bucket, p Photo) (string, error) { + err := b.Validate() + if err != nil { + return "", err + } + err = p.Validate() + if err != nil { + return "", err + } + + url := s.baseBucket(b) + url.Path += p.Path() + + req, err := http.NewRequest("PUT", url.String(), nil) + if err != nil { + return "", err + } + + signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Write/time.Second)) + return signedReq.URL.String(), nil +}