diff --git a/README.md b/README.md index 6e824a1..2dfeafc 100644 --- a/README.md +++ b/README.md @@ -3,48 +3,104 @@ A photo bucket management suite. -There are two modes of operation: -- Domain - - Buckets are exactly equal to their domain names - - `unset MINIO_DOMAIN` -- Subdomain - - Buckets are named after subdomains - - `export MINIO_DOMAIN=your.domain` +## Bucket Format + +- `photo/[filename]` + - Original photo. Name conflict resolution (excluding extension) must be handled client-side. +- `preview/[filename]_h[height]q[quality].[format]` + - Preview photos at lower resolutions and quality. + - Generator: preview +- `photometadata/[filename]/size` + - Original photo size, JSON + - Generator: preview +- `photometadata/[filename]/date` + - Original photo date taken, unix timestamp + - Generator: preview +- `photometadata/[filename]/title` + - Photo title +- `metadata/title` + - Photo album title +- `metadata/description` + - Photo album description, markdown +- `metadata/ordering` + - Manual sort ordering, newline separated +- `internal/control` + - Access control settings, JSON ## `admin` -Create new buckets. Standalone tool. - -## `control` - -Implement access controls by signing or proxying requests. +- Standalone tool to manage an S3-like account + - In-browser handling of: + - ListBuckets + - MakeBucket + - RemoveBucket + - Setup photo bucket + - Uploads initial metadata + - SetBucketPolicy that matches + - Adds credentials to database. + - Disconnect photo bucket + - Removes credentials from database ### Operations -#### `GET /list?bucket=BUCKET&auth=TOKEN` +#### `PUT /credential?bucket=BUCKET` -1. Consult the bucket for metadata.json -2. Get list access method for the bucket -3. Validate the token against the access method -4. Return ListObjectsV2 for prefix `photo/` - - Can also 307 redirect to the bucket read URL, if is public readable +1. Ensure credentials work on BUCKET +2. Write credentials to KV store -#### `GET /read?bucket=BUCKET&auth=TOKEN&object=OBJECTNAME` +#### `DELETE /credential?bucket=BUCKET` -1. Consult the bucket for metadata.json +1. Ensure credentials work on BUCKET +2. Write credentials to KV store + +### Data + +Credentials must be passed as: +``` +{ + "access_key": string, + "secret_key": string, + "region": string +} +``` + +## `control` + +Implement access controls by signing requests, using credentials in database. + +### Operations + +#### `GET /sign?bucket=BUCKET&token=TOKEN&request=REQUEST` + +1. Consult the bucket for `metadata/control` with credentials 2. Get read access method for the bucket 3. Validate the token against the access method -4. Validate that OBJECTNAME starts with `photo/` +4. Validate REQUEST is an acceptable request + - ListObjectsV2, GetObject + - Not `internal/` 5. If necessary, presign an object URL for 4 days - - Cache presigned URLs for 2 days in memory/Redis + - Cache presigned URLs for 2 days in memory/Redis 6. 307 redirect to presigned URL -#### `PUT /write?bucket=BUCKET&auth=TOKEN&object=OBJECTNAME` +#### `PUT /sign?bucket=BUCKET&token=TOKEN&request=REQUEST` -1. Consult the bucket for metadata.json +1. Consult the bucket for `metadata/control` with credentials 2. Get write access method for the bucket 3. Validate the token against the access method -4. Validate that OBJECTNAME starts with `photo/` +4. Validate REQUEST is an acceptable request + - PutObject + - Not `internal/` +5. If necessary, presign an object URL for 30 minutes +6. 307 redirect to presigned URL + +#### `DELETE /sign?bucket=BUCKET&token=TOKEN&request=REQUEST` + +1. Consult the bucket for `metadata/control` with credentials +2. Get write access method for the bucket +3. Validate the token against the access method +4. Validate REQUEST is an acceptable request + - RemoveObject + - Not `internal/` 5. If necessary, presign an object URL for 30 minutes 6. 307 redirect to presigned URL @@ -58,27 +114,24 @@ The read/write token is checked against a simple string defined in the bucket. Recommended IDP: [dex](https://github.com/dexidp/dex) -The read/write operation is gated by a signed key corresponding to allowed -users defined in the bucket. +The read/write operation is gated by a signed key corresponding to allowed users defined in the bucket. ## `web` -Generates the web interface for a photo bucket. Also updates the shared asset bucket on start. +Generates the web interface for a photo bucket. Also serves up the shared assets. ### Operations -#### `POST /webhook` #### `POST /update?bucket=BUCKET` Regenerate and upload `index.html` and `manage/index.html` to bucket. ## `preview` -Generate previews from photo buckets. Registers webhooks. +Generate previews from photo buckets. ### Operations -#### `POST /webhook` #### `POST /update?bucket=BUCKET&photo=OBJECT` 1. Perform preview generation using libvips (maybe limit?) @@ -86,7 +139,7 @@ Generate previews from photo buckets. Registers webhooks. ## `proxy` -Reverse proxies buckets to the minio endpoint, as a substitute for the AWS S3 website hosting features. Serves up `index.html` when URLs end in a slash. +Reverse proxies buckets to the minio endpoint, as a substitute for AWS S3 website hosting features. Serves up `index.html` when URLs end in a slash. In production, [replace this with Nginx](https://docs.min.io/docs/setup-nginx-proxy-with-minio). diff --git a/build/web/Dockerfile b/build/web/Dockerfile index 291f6e3..cd2c3fb 100644 --- a/build/web/Dockerfile +++ b/build/web/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine AS build +FROM golang:buster AS build RUN mkdir /src /dist @@ -10,7 +10,7 @@ ENV CGO_ENABLED=0 RUN go build -o /dist/web -FROM scratch +FROM debian:buster COPY --from=build /dist/web /web diff --git a/cmd/control/control.go b/cmd/control/control.go index b44b4e7..a788646 100644 --- a/cmd/control/control.go +++ b/cmd/control/control.go @@ -3,114 +3,90 @@ package main import ( "fmt" "net/http" - "strconv" "time" "git.makerforce.io/photos/photos/internal/httphelpers" lib "git.makerforce.io/photos/photos/pkg/bucket" + "git.makerforce.io/photos/photos/pkg/credentials" + "git.makerforce.io/photos/photos/pkg/signer" ) -var signer *lib.Signer +var creds *credentials.Client +var sig *signer.Signer func main() { - // Setup bucket signer var err error - signer, err = lib.NewSignerFromEnv() + // Setup bucket credential source + creds, err = credentials.NewClientFromEnv() + if err != nil { + panic(err) + } + // Setup bucket signer + sig, err = signer.NewSignerFromEnv() + if err != nil { + panic(err) + } server := &http.Server{ Addr: ":8000", ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, } - http.HandleFunc("/list", list) - http.HandleFunc("/read", read) - http.HandleFunc("/write", write) + http.HandleFunc("/sign", sign) err = server.ListenAndServe() if err != nil { panic(err) } } -func list(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { +func sign(w http.ResponseWriter, req *http.Request) { + var err error + + bucket := lib.Bucket(req.FormValue("bucket")) + token := req.FormValue("token") + request := SafePathable(req.FormValue("request")) + + err = bucket.Validate() + if err != nil { + err := fmt.Errorf("%w: %v", httphelpers.ErrorBadRequest, err.Error()) + httphelpers.ErrorResponse(w, err) + return + } + err = request.Validate() + if err != nil { + err := fmt.Errorf("%w: %v", httphelpers.ErrorBadRequest, err.Error()) + httphelpers.ErrorResponse(w, err) + return + } + + var cred credentials.Credential + if token == "todo" { + cred, err = creds.Get(bucket) + if err != nil { + err := fmt.Errorf("%w: error getting credentials: %v", httphelpers.ErrorBadRequest, err.Error()) + httphelpers.ErrorResponse(w, err) + return + } + } + + var signedReq *http.Request + unsignedReq, err := http.NewRequest(req.Method, bucket.URL(request).String(), nil) + if err != nil { + err := fmt.Errorf("%w: error creating request: %v", httphelpers.ErrorBadRequest, err.Error()) + httphelpers.ErrorResponse(w, err) + return + } + + if req.Method == http.MethodGet { + signedReq = sig.PreSignRead(unsignedReq, cred) + } else if req.Method == http.MethodPost || req.Method == http.MethodPut || req.Method == http.MethodDelete { + signedReq = sig.PreSignRead(unsignedReq, cred) + } else { err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) httphelpers.ErrorResponse(w, err) return } - bucket := lib.Bucket(req.FormValue("bucket")) - startAfter := req.FormValue("start-after") - maxKeysString := req.FormValue("max-keys") - if maxKeysString == "" { - maxKeysString = "4000" - } - maxKeys, err := strconv.Atoi(maxKeysString) // Let minio handle error - - url, err := signer.GetBucketPhotos(bucket, startAfter, maxKeys) - if err != nil { - httphelpers.ErrorResponse(w, err) - return - } - - w.Header().Add("Location", url) + w.Header().Add("Location", signedReq.URL.String()) w.WriteHeader(http.StatusTemporaryRedirect) } - -func read(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet { - err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) - httphelpers.ErrorResponse(w, err) - return - } - - bucket := lib.Bucket(req.FormValue("bucket")) - photo := lib.Photo(req.FormValue("object")) - preview := lib.Preview(req.FormValue("object")) - - if photo.Validate() == nil { - url, err := signer.GetPhoto(bucket, photo) - if err != nil { - httphelpers.ErrorResponse(w, err) - return - } - - w.Header().Add("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) - return - } else if preview.Validate() == nil { - url, err := signer.GetPreview(bucket, preview) - if err != nil { - httphelpers.ErrorResponse(w, err) - return - } - - w.Header().Add("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) - return - } - httphelpers.ErrorResponse(w, httphelpers.ErrorBadRequest) -} - -func write(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodPut { - err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) - httphelpers.ErrorResponse(w, err) - return - } - - bucket := lib.Bucket(req.FormValue("bucket")) - photo := lib.Photo(req.FormValue("object")) - - if photo.Validate() == nil { - url, err := signer.PutPhoto(bucket, photo) - if err != nil { - httphelpers.ErrorResponse(w, err) - return - } - - w.Header().Add("Location", url) - w.WriteHeader(http.StatusTemporaryRedirect) - return - } - httphelpers.ErrorResponse(w, httphelpers.ErrorBadRequest) -} diff --git a/cmd/control/pathable.go b/cmd/control/pathable.go new file mode 100644 index 0000000..98e4582 --- /dev/null +++ b/cmd/control/pathable.go @@ -0,0 +1,27 @@ +package main + +import ( + "errors" + "net/url" + "strings" +) + +type SafePathable string + +var ErrorInvalidRequest = errors.New("invalid request") + +func (s SafePathable) Validate() error { + if strings.HasPrefix(string(s), "/") { + return ErrorInvalidRequest + } + if strings.HasPrefix(string(s), "internal/") { + return ErrorInvalidRequest + } + return nil +} + +func (s SafePathable) Path() *url.URL { + return &url.URL{ + Path: string(s), + } +} diff --git a/deployments/experiments.yml b/deployments/experiments.yml index 79249d8..7a6614c 100644 --- a/deployments/experiments.yml +++ b/deployments/experiments.yml @@ -42,6 +42,118 @@ spec: - protocol: TCP port: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: photos-control + labels: + app: photos-control +spec: + replicas: 1 + selector: + matchLabels: + app: photos-control + template: + metadata: + labels: + app: photos-control + spec: + containers: + - name: control + image: registry.makerforce.io/photos/control:dev + imagePullPolicy: Always + env: + - name: MINIO_ENDPOINT + value: api.ambrose.photos:9000 + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: minio + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: minio + - name: MINIO_REGION_NAME + value: "sgp1" + - name: MINIO_ENDPOINT_SECURE + value: "false" + - name: MINIO_CREDENTIALS_BUCKET + value: "credentials" + ports: + - containerPort: 8000 + +--- +apiVersion: v1 +kind: Service +metadata: + name: photos-control +spec: + selector: + app: photos-control + ports: + - protocol: TCP + port: 80 + targetPort: 8000 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: photos-preview + labels: + app: photos-preview +spec: + replicas: 1 + selector: + matchLabels: + app: photos-preview + template: + metadata: + labels: + app: photos-preview + spec: + containers: + - name: preview + image: registry.makerforce.io/photos/preview:dev + imagePullPolicy: Always + env: + - name: MINIO_ENDPOINT + value: api.ambrose.photos:9000 + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: accesskey + name: minio + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: secretkey + name: minio + - name: MINIO_REGION_NAME + value: "sgp1" + - name: MINIO_ENDPOINT_SECURE + value: "false" + - name: MINIO_CREDENTIALS_BUCKET + value: "credentials" + ports: + - containerPort: 8003 + +--- +apiVersion: v1 +kind: Service +metadata: + name: photos-preview +spec: + selector: + app: photos-preview + ports: + - protocol: TCP + port: 80 + targetPort: 8003 + --- apiVersion: apps/v1 kind: Deployment @@ -66,10 +178,6 @@ spec: env: - name: MINIO_ENDPOINT value: api.ambrose.photos:9000 - - name: MINIO_DOMAIN - value: "" - - name: MINIO_ENDPOINT_SECURE - value: "false" - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: @@ -80,6 +188,12 @@ spec: secretKeyRef: key: secretkey name: minio + - name: MINIO_REGION_NAME + value: "sgp1" + - name: MINIO_ENDPOINT_SECURE + value: "false" + - name: MINIO_CREDENTIALS_BUCKET + value: "credentials" - name: WEB_ENDPOINT value: "http://web.ambrose.photos" ports: diff --git a/internal/s3utils/hostpath.go b/internal/s3utils/hostpath.go deleted file mode 100644 index add08e5..0000000 --- a/internal/s3utils/hostpath.go +++ /dev/null @@ -1,8 +0,0 @@ -package s3utils - -func PathFromHost(path, host string) string { - if len(host) > 0 { - return "/" + host + path - } - return path -} diff --git a/pkg/bucket/bucket.go b/pkg/bucket/bucket.go index 2adfa91..2ae8ec8 100644 --- a/pkg/bucket/bucket.go +++ b/pkg/bucket/bucket.go @@ -3,16 +3,26 @@ package bucket import ( "errors" "fmt" + "net/url" + "strings" ) type Bucket string -const bucketMetadataObject = "metadata.json" +const bucketMetadataPrefix = "metadata/" var ErrorInvalidBucket = errors.New("invalid bucket") func (b Bucket) Validate() error { - if len(b) < 1 || b == "meta" { + // Ensure is URL + u, err := url.ParseRequestURI(string(b)) + if err != nil { + return fmt.Errorf("%w: %w: %v", ErrorInvalidBucket, err, b) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("%w: %v", ErrorInvalidBucket, b) + } + if u.Host == "" || !strings.HasSuffix(u.Path, "/") { return fmt.Errorf("%w: %v", ErrorInvalidBucket, b) } return nil @@ -22,7 +32,45 @@ func (b Bucket) String() string { return string(b) } -type BucketMetadata struct { - Title string `json:"title"` - PreviewOptions []PreviewOption `json:"preview_options"` +func (b Bucket) URL(p Pathable) *url.URL { + u, _ := url.ParseRequestURI(string(b)) + path := p.Path() + u.Path = pathJoin(u.Path, path.Path) + u.RawQuery = path.RawQuery + return u +} + +func pathJoin(paths ...string) string { + cleanedPaths := make([]string, 0, len(paths)) + for i, path := range paths { + cleanedPath := strings.TrimPrefix(path, "/") + if i != len(paths)-1 { + cleanedPath = strings.TrimSuffix(cleanedPath, "/") + } + if i == 0 && len(cleanedPath) == 0 { + continue + } + cleanedPaths = append(cleanedPaths, cleanedPath) + } + return "/" + strings.Join(cleanedPaths, "/") +} + +type Pathable interface { + Path() *url.URL +} + +type BucketMetadata string + +func (b Bucket) Metadata(name string) BucketMetadata { + return BucketMetadata(bucketMetadataPrefix + name) +} + +func (m BucketMetadata) String() string { + return string(m) +} + +func (m BucketMetadata) Path() *url.URL { + return &url.URL{ + Path: string(m), + } } diff --git a/pkg/bucket/bucket_test.go b/pkg/bucket/bucket_test.go new file mode 100644 index 0000000..37e4082 --- /dev/null +++ b/pkg/bucket/bucket_test.go @@ -0,0 +1,79 @@ +package bucket + +import ( + "testing" +) + +func TestBucket_Validate(t *testing.T) { + var got error + + got = Bucket("http://api.url").Validate() + if got == nil { + t.Errorf("Bucket.Validate() = %v; want non-nil", got) + } + + got = Bucket("http://api.url/").Validate() + if got != nil { + t.Errorf("Bucket.Validate() = %v; want nil", got) + } + + got = Bucket("http://api.url/bucket").Validate() + if got == nil { + t.Errorf("Bucket.Validate() = %v; want non-nil", got) + } + + got = Bucket("http://api.url/bucket/").Validate() + if got != nil { + t.Errorf("Bucket.Validate() = %v; want nil", got) + } + + got = Bucket("http://api.url/bucket/path/").Validate() + if got != nil { + t.Errorf("Bucket.Validate() = %v; want nil", got) + } +} + +func TestBucket_URL(t *testing.T) { + var got, want string + var b Bucket + + b = Bucket("http://api.url/bucket/") + + got = b.URL(Photo("/photo/hello")).String() + want = "http://api.url/bucket/photo/hello" + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } + + got = b.URL(Preview("preview/hello")).String() + want = "http://api.url/bucket/preview/hello" + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } + + got = b.URL(NewList("photo/", 1000, "name")).String() + want = "http://api.url/bucket/?delimiter=&encoding-type=url&list-type=2&max-keys=1000&metadata=true&prefix=photo%2F&start-after=name" + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } + + b = Bucket("http://bucket.api.url") + + got = b.URL(Photo("/photo/hello")).String() + want = "http://bucket.api.url/photo/hello" + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } + + got = b.URL(Preview("preview/hello")).String() + want = "http://bucket.api.url/preview/hello" + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } + + got = b.URL(NewList("photo/", 1000, "name")).String() + want = "http://bucket.api.url" + NewList("photo/", 1000, "name").Path().String() + if got != want { + t.Errorf("Bucket.URL(Photo) = %v; want %v", got, want) + } +} diff --git a/pkg/bucket/client.go b/pkg/bucket/client.go deleted file mode 100644 index 4904e2d..0000000 --- a/pkg/bucket/client.go +++ /dev/null @@ -1,205 +0,0 @@ -package bucket - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "os" - - "github.com/minio/minio-go/v6" -) - -type Client struct { - *minio.Client -} - -func NewClient(endpoint, accessKey, secretKey, regionName string, endpointSecure bool) (*Client, error) { - m, err := minio.NewWithRegion(endpoint, accessKey, secretKey, endpointSecure, regionName) - if err != nil { - return nil, err - } - - client := &Client{ - Client: m, - } - return client, nil -} - -func NewClientFromEnv() (*Client, error) { - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") - regionName := os.Getenv("MINIO_REGION_NAME") - endpoint := os.Getenv("MINIO_ENDPOINT") - endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true" - return NewClient(endpoint, accessKey, secretKey, regionName, endpointSecure) -} - -func (c *Client) GetBucketMetadata(b Bucket) (BucketMetadata, error) { - err := b.Validate() - if err != nil { - return BucketMetadata{}, err - } - - o, err := c.GetObject(b.String(), bucketMetadataObject, minio.GetObjectOptions{}) - if err != nil { - return BucketMetadata{}, err - } - defer o.Close() - buf, err := ioutil.ReadAll(o) - if err != nil { - return BucketMetadata{}, err - } - - m := BucketMetadata{} - err = json.Unmarshal(buf, &m) - if err != nil { - return BucketMetadata{}, err - } - return m, nil -} - -func (c *Client) PutBucketMetadata(b Bucket, m BucketMetadata) error { - err := b.Validate() - if err != nil { - return err - } - - buf, err := json.MarshalIndent(m, "", "\t") - if err != nil { - return err - } - - _, err = c.PutObject(b.String(), bucketMetadataObject, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: "application/json", - }) - if err != nil { - return err - } - return nil -} - -func (c *Client) PutBucketWeb(b Bucket, buf []byte) error { - err := b.Validate() - if err != nil { - return err - } - - _, err = c.PutObject(b.String(), "index.html", bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: "text/html", - StorageClass: "REDUCED_REDUNDANCY", - }) - if err != nil { - return err - } - return nil -} - -func (c *Client) GetPhotos(b Bucket) ([]Photo, error) { - err := b.Validate() - if err != nil { - return nil, err - } - - keys := make([]Photo, 0) - - doneCh := make(chan struct{}) - objCh := c.ListObjectsV2(b.String(), photoPrefix, true, doneCh) - for obj := range objCh { - if obj.Err != nil { - return nil, obj.Err - } - keys = append(keys, Photo(obj.Key)) - } - - return keys, nil -} - -func (c *Client) GetPhoto(b Bucket, p Photo) ([]byte, error) { - err := b.Validate() - if err != nil { - return nil, err - } - err = p.Validate() - if err != nil { - return nil, err - } - - o, err := c.GetObject(b.String(), p.String(), minio.GetObjectOptions{}) - if err != nil { - return nil, err - } - defer o.Close() - return ioutil.ReadAll(o) -} - -func (c *Client) GetPhotoMetadata(b Bucket, p Photo) (PhotoMetadata, error) { - err := b.Validate() - if err != nil { - return PhotoMetadata{}, err - } - err = p.Validate() - if err != nil { - return PhotoMetadata{}, err - } - - o, err := c.GetObject(b.String(), p.MetadataString(), minio.GetObjectOptions{}) - if err != nil { - return PhotoMetadata{}, err - } - defer o.Close() - buf, err := ioutil.ReadAll(o) - if err != nil { - return PhotoMetadata{}, err - } - - m := PhotoMetadata{} - err = json.Unmarshal(buf, &m) - if err != nil { - return PhotoMetadata{}, err - } - return m, nil -} - -func (c *Client) PutPhotoMetadata(b Bucket, p Photo, m PhotoMetadata) error { - err := b.Validate() - if err != nil { - return err - } - err = p.Validate() - if err != nil { - return err - } - - buf, err := json.MarshalIndent(m, "", "\t") - if err != nil { - return err - } - - _, err = c.PutObject(b.String(), p.MetadataString(), bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: "application/json", - }) - if err != nil { - return err - } - return nil -} - -func (c *Client) PutPreview(b Bucket, p Preview, buf []byte) error { - err := b.Validate() - if err != nil { - return err - } - err = p.Validate() - if err != nil { - return err - } - - _, err = c.PutObject(b.String(), p.String(), bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: p.Format().ContentType(), - StorageClass: "REDUCED_REDUNDANCY", - }) - if err != nil { - return err - } - return nil -} diff --git a/pkg/bucket/list.go b/pkg/bucket/list.go new file mode 100644 index 0000000..8cc6f7f --- /dev/null +++ b/pkg/bucket/list.go @@ -0,0 +1,37 @@ +package bucket + +import ( + "fmt" + "net/url" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +type List struct { + prefix string + maxKeys int + startAfter string +} + +func NewList(prefix string, maxKeys int, startAfter string) List { + return List{ + prefix: prefix, + maxKeys: maxKeys, + startAfter: startAfter, + } +} + +func (l List) Path() *url.URL { + params := make(url.Values) + params.Set("list-type", "2") + params.Set("metadata", "true") + params.Set("encoding-type", "url") + params.Set("prefix", l.prefix) + params.Set("delimiter", "") + params.Set("max-keys", fmt.Sprintf("%d", l.maxKeys)) + params.Set("start-after", l.startAfter) + return &url.URL{ + Path: "/", + RawQuery: s3utils.QueryEncode(params), + } +} diff --git a/pkg/bucket/photo.go b/pkg/bucket/photo.go index a5f4cca..c5ba525 100644 --- a/pkg/bucket/photo.go +++ b/pkg/bucket/photo.go @@ -10,6 +10,7 @@ import ( type Photo string const photoPrefix = "photo/" +const photoMetadataPrefix = "photometadata/" var ErrorInvalidPhoto = errors.New("invalid photo") @@ -17,6 +18,9 @@ func (p Photo) Validate() error { if !strings.HasPrefix(string(p), photoPrefix) { return fmt.Errorf("%w: %v", ErrorInvalidPhoto, p) } + if strings.HasSuffix(string(p), "/") { + return fmt.Errorf("%w: %v", ErrorInvalidPhoto, p) + } return nil } @@ -24,23 +28,25 @@ func (p Photo) String() string { return string(p) } -func (p Photo) Path() string { - u := url.URL{ +func (p Photo) Path() *url.URL { + return &url.URL{ Path: string(p), } - return u.String() } -const photoMetadataPrefix = "photometa/" +type PhotoMetadata string -func (p Photo) MetadataString() string { +func (p Photo) Metadata(name string) PhotoMetadata { objectBase := strings.Replace(string(p), photoPrefix, photoMetadataPrefix, 1) - objectBase += ".metadata.json" - return objectBase + return PhotoMetadata(objectBase + "/" + name) } -type PhotoMetadata struct { - Title string `json:"title"` - Width int `json:"width"` - Height int `json:"height"` +func (m PhotoMetadata) String() string { + return string(m) +} + +func (m PhotoMetadata) Path() *url.URL { + return &url.URL{ + Path: string(m), + } } diff --git a/pkg/bucket/photopreview.go b/pkg/bucket/photopreview.go index 4f4aa16..44b0a5f 100644 --- a/pkg/bucket/photopreview.go +++ b/pkg/bucket/photopreview.go @@ -19,6 +19,9 @@ func (p Preview) Validate() error { if !strings.HasPrefix(string(p), previewPrefix) { return fmt.Errorf("%w: %v", ErrorInvalidPreview, p) } + if strings.HasSuffix(string(p), "/") { + return fmt.Errorf("%w: %v", ErrorInvalidPreview, p) + } return nil } @@ -26,11 +29,10 @@ func (p Preview) String() string { return string(p) } -func (p Preview) Path() string { - u := url.URL{ +func (p Preview) Path() *url.URL { + return &url.URL{ Path: string(p), } - return u.String() } func (p Preview) Format() PhotoFormat { @@ -93,7 +95,7 @@ func (p Photo) GetPreview(option PreviewOption) Preview { extIndex := strings.LastIndex(objectBase, ".") base := objectBase[:extIndex] res := fmt.Sprintf("_h%dq%d", option.Height, option.Quality) - exts, err := mime.ExtensionsByType(option.Format.ContentType()) + exts, err := mime.ExtensionsByType(option.Format.ContentType()) // TODO: replace with hardcoded list if err != nil { panic(err) } diff --git a/pkg/bucket/signer.go b/pkg/bucket/signer.go deleted file mode 100644 index c2ce2de..0000000 --- a/pkg/bucket/signer.go +++ /dev/null @@ -1,198 +0,0 @@ -package bucket - -import ( - "errors" - "net" - "net/http" - "net/url" - "os" - "strconv" - "time" - - internal_s3utils "git.makerforce.io/photos/photos/internal/s3utils" - "github.com/minio/minio-go/v6/pkg/s3utils" - "github.com/minio/minio-go/v6/pkg/signer" -) - -type Signer struct { - accessKey string - secretKey string - regionName string - bucketSecure bool - expirations Expirations -} - -type Expirations struct { - // Expiration time for list and read in time.Duration - Read time.Duration - // Expiration time for write in time.Duration - Write time.Duration -} - -var ErrorExpirationTooLow = errors.New("expiration time too low") - -func NewSigner(accessKey, secretKey, regionName string, bucketSecure bool, expirations Expirations) (*Signer, error) { - if expirations.Read == 0 { - expirations.Read = 30 * time.Minute - } - if expirations.Write == 0 { - expirations.Write = 5 * time.Minute - } - if expirations.Read < time.Second || expirations.Write < time.Second { - return nil, ErrorExpirationTooLow - } - signer := &Signer{ - accessKey: accessKey, - secretKey: secretKey, - regionName: regionName, - bucketSecure: bucketSecure, - expirations: expirations, - } - return signer, nil -} - -func NewSignerFromEnv() (*Signer, error) { - accessKey := os.Getenv("MINIO_ACCESS_KEY") - secretKey := os.Getenv("MINIO_SECRET_KEY") - regionName := os.Getenv("MINIO_REGION_NAME") - bucketSecure := os.Getenv("BUCKET_SECURE") == "true" - expirationRead, _ := strconv.Atoi(os.Getenv("EXPIRATION_READ")) - expirationWrite, _ := strconv.Atoi(os.Getenv("EXPIRATION_WRITE")) - expirations := Expirations{Read: time.Duration(expirationRead), Write: time.Duration(expirationWrite)} - return NewSigner(accessKey, secretKey, regionName, bucketSecure, expirations) -} - -func (s *Signer) baseBucket(b Bucket) url.URL { - url := url.URL{ - Scheme: "http", - Host: b.String(), - Path: "/", - } - if s.bucketSecure { - url.Scheme = "https" - } - return url -} - -func (s *Signer) preSignV4(req http.Request) http.Request { - domain := os.Getenv("MINIO_DOMAIN") - - // Validate host - host, _, err := net.SplitHostPort(req.Host) - if err != nil { - host = req.Host - } - originalPath := req.URL.Path - - if len(domain) < 1 { - // If we are using domains, rewrite the host into path - req.URL.Path = internal_s3utils.PathFromHost(req.URL.Path, host) - } - // If we are using subdomains, sign the direct URL - signedReq := signer.PreSignV4( - req, - s.accessKey, s.secretKey, "", - "sgp1", - int64(s.expirations.Read/time.Second), - ) - signedReq.URL.Path = originalPath - return *signedReq -} - -func listBucketParams(prefix, startAfter string, maxKeys int) url.Values { - params := make(url.Values) - params.Set("list-type", "2") - params.Set("metadata", "true") - params.Set("encoding-type", "url") - params.Set("prefix", prefix) - params.Set("delimiter", "") - params.Set("max-keys", strconv.Itoa(maxKeys)) - params.Set("start-after", startAfter) - return params -} - -func (s *Signer) GetBucketPhotos(b Bucket, startAfter string, maxKeys int) (string, error) { - err := b.Validate() - if err != nil { - return "", err - } - - url := s.baseBucket(b) - url.Path += "/" - params := listBucketParams(photoPrefix, startAfter, maxKeys) - url.RawQuery = s3utils.QueryEncode(params) - - req, err := http.NewRequest("GET", url.String(), nil) - if err != nil { - return "", err - } - - signedReq := s.preSignV4(*req) - return signedReq.URL.String(), nil -} - -func (s *Signer) GetPhoto(b Bucket, p Photo) (string, error) { - err := b.Validate() - if err != nil { - return "", err - } - err = p.Validate() - if err != nil { - return "", err - } - - url := s.baseBucket(b) - url.Path += p.Path() - - req, err := http.NewRequest("GET", url.String(), nil) - if err != nil { - return "", err - } - - signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Read/time.Second)) - return signedReq.URL.String(), nil -} - -func (s *Signer) PutPhoto(b Bucket, p Photo) (string, error) { - err := b.Validate() - if err != nil { - return "", err - } - err = p.Validate() - if err != nil { - return "", err - } - - url := s.baseBucket(b) - url.Path += p.Path() - - req, err := http.NewRequest("PUT", url.String(), nil) - if err != nil { - return "", err - } - - signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Write/time.Second)) - return signedReq.URL.String(), nil -} - -func (s *Signer) GetPreview(b Bucket, p Preview) (string, error) { - err := b.Validate() - if err != nil { - return "", err - } - err = p.Validate() - if err != nil { - return "", err - } - - url := s.baseBucket(b) - url.Path += p.Path() - - req, err := http.NewRequest("GET", url.String(), nil) - if err != nil { - return "", err - } - - signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Read/time.Second)) - return signedReq.URL.String(), nil -} diff --git a/pkg/credentials/client.go b/pkg/credentials/client.go new file mode 100644 index 0000000..a7087cf --- /dev/null +++ b/pkg/credentials/client.go @@ -0,0 +1,87 @@ +package credentials + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "log" + "os" + "strings" + + "git.makerforce.io/photos/photos/pkg/bucket" + "github.com/minio/minio-go/v6" +) + +type Client struct { + *minio.Client + Bucket string +} + +func NewClient(endpoint, accessKey, secretKey, regionName string, endpointSecure bool, bucket string) (*Client, error) { + m, err := minio.NewWithRegion(endpoint, accessKey, secretKey, endpointSecure, regionName) + if err != nil { + return nil, err + } + + client := &Client{ + Client: m, + Bucket: bucket, + } + return client, nil +} + +func NewClientFromEnv() (*Client, error) { + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + regionName := os.Getenv("MINIO_REGION_NAME") + endpoint := os.Getenv("MINIO_ENDPOINT") + endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true" + bucket := os.Getenv("MINIO_CREDENTIALS_BUCKET") + return NewClient(endpoint, accessKey, secretKey, regionName, endpointSecure, bucket) +} + +func (c *Client) Get(bucket bucket.Bucket) (Credential, error) { + normalizedBucket := normalize(bucket.String()) + log.Printf("Getting credentials for %s using %s %s", normalizedBucket, c.EndpointURL(), c.Bucket) + + o, err := c.GetObject(c.Bucket, normalizedBucket, minio.GetObjectOptions{}) + if err != nil { + return Credential{}, err + } + defer o.Close() + buf, err := ioutil.ReadAll(o) + if err != nil { + return Credential{}, err + } + + cred := Credential{} + err = json.Unmarshal(buf, &cred) + if err != nil { + return Credential{}, err + } + return cred, nil +} + +func (c *Client) Set(bucket bucket.Bucket, cred Credential) error { + normalizedBucket := normalize(bucket.String()) + log.Printf("Setting credentials for %s using %s %s", normalizedBucket, c.EndpointURL(), c.Bucket) + + buf, err := json.MarshalIndent(cred, "", "\t") + if err != nil { + return err + } + + _, err = c.PutObject(c.Bucket, normalizedBucket, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "application/json", + }) + if err != nil { + return err + } + return nil +} + +func normalize(bucket string) string { + nodouble := strings.Replace(bucket, "//", "", 1) + nolastslash := strings.TrimSuffix(nodouble, "/") + return nolastslash +} diff --git a/pkg/credentials/types.go b/pkg/credentials/types.go new file mode 100644 index 0000000..ac06938 --- /dev/null +++ b/pkg/credentials/types.go @@ -0,0 +1,7 @@ +package credentials + +type Credential struct { + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + Region string `json:"region"` +} diff --git a/pkg/signer/signer.go b/pkg/signer/signer.go new file mode 100644 index 0000000..c1b404e --- /dev/null +++ b/pkg/signer/signer.go @@ -0,0 +1,76 @@ +package signer + +import ( + "errors" + "net/http" + "os" + "strconv" + "time" + + "git.makerforce.io/photos/photos/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/signer" +) + +type Signer struct { + expirations Expirations +} + +type Expirations struct { + // Expiration time for list and read in time.Duration + Read time.Duration + // Expiration time for write in time.Duration + Write time.Duration +} + +var ErrorExpirationTooLow = errors.New("expiration time too low") + +func NewSigner(expirations Expirations) (*Signer, error) { + if expirations.Read == 0 { + expirations.Read = 30 * time.Minute + } + if expirations.Write == 0 { + expirations.Write = 5 * time.Minute + } + if expirations.Read < time.Second || expirations.Write < time.Second { + return nil, ErrorExpirationTooLow + } + return &Signer{ + expirations: expirations, + }, nil +} + +func NewSignerFromEnv() (*Signer, error) { + expirationRead, _ := strconv.Atoi(os.Getenv("EXPIRATION_READ")) + expirationWrite, _ := strconv.Atoi(os.Getenv("EXPIRATION_WRITE")) + expirations := Expirations{Read: time.Duration(expirationRead), Write: time.Duration(expirationWrite)} + return NewSigner(expirations) +} + +func (s *Signer) PreSignRead(req *http.Request, cred credentials.Credential) *http.Request { + signedReq := signer.PreSignV4( + *req, + cred.AccessKey, cred.SecretKey, "", + cred.Region, + int64(s.expirations.Read/time.Second), + ) + return signedReq +} + +func (s *Signer) PreSignWrite(req *http.Request, cred credentials.Credential) *http.Request { + signedReq := signer.PreSignV4( + *req, + cred.AccessKey, cred.SecretKey, "", + cred.Region, + int64(s.expirations.Write/time.Second), + ) + return signedReq +} + +func (s *Signer) Sign(req *http.Request, cred credentials.Credential) *http.Request { + signedReq := signer.SignV4( + *req, + cred.AccessKey, cred.SecretKey, "", + cred.Region, + ) + return signedReq +} diff --git a/test.ambrose.photos b/test.ambrose.photos new file mode 100644 index 0000000..03124c3 --- /dev/null +++ b/test.ambrose.photos @@ -0,0 +1,5 @@ +{ + "access_key": "azCdtns6MMayxqEWu0UrdDZp", + "secret_key": "W3sJ65RPgr3VzrOeH03nFGvUI5T8ZpESY5OUnvJz6c8Uv3SaIZOA22MboAf4NbEz", + "region": "sgp1" +} diff --git a/web/manage/src/Photo.svelte b/web/manage/src/Photo.svelte index ba49c3d..2a2ce30 100644 --- a/web/manage/src/Photo.svelte +++ b/web/manage/src/Photo.svelte @@ -1,7 +1,12 @@