diff --git a/.gitignore b/.gitignore index c322087..c2d7f83 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ cmd/control/control cmd/web/web cmd/indexer/indexer cmd/thumbnail/thumbnail + +env diff --git a/README.md b/README.md index fbf87b4..4d24afb 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Implement access controls by signing or proxying requests. #### `GET /list?bucket=BUCKET&auth=TOKEN` 1. Consult the bucket for metadata.json -2. Get read access method for the bucket +2. Get list access method for the bucket 3. Validate the token against the access method 4. Return ListObjectsV2 for prefix `photo/` - Can also 307 redirect to the bucket read URL, if is public readable @@ -82,9 +82,9 @@ Generate thumbnails from photo buckets. Registers webhooks. ### Operations #### `POST /webhook` -#### `POST /update?bucket=BUCKET&object=OBJECT` +#### `POST /update?bucket=BUCKET&photo=OBJECT` -1. Perform thumbnail generation using libvips in a pool queue. +1. Perform thumbnail generation using libvips (maybe limit?) 2. Block until done diff --git a/cmd/control/control.go b/cmd/control/control.go index 5d94f7b..c0d5944 100644 --- a/cmd/control/control.go +++ b/cmd/control/control.go @@ -20,12 +20,16 @@ func main() { secretKey = os.Getenv("MINIO_SECRET_KEY") server := &http.Server{ + Addr: ":8000", ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, } http.HandleFunc("/list", hello) http.HandleFunc("/write", hello2) - server.ListenAndServe() + err := server.ListenAndServe() + if err != nil { + panic(err) + } } func hello(w http.ResponseWriter, req *http.Request) { diff --git a/cmd/thumbnail/thumbnail.go b/cmd/thumbnail/thumbnail.go index cbfc4cf..240614c 100644 --- a/cmd/thumbnail/thumbnail.go +++ b/cmd/thumbnail/thumbnail.go @@ -1,97 +1,146 @@ package main import ( - "bytes" - "io/ioutil" + "fmt" "log" "net/http" - "os" + "sync" "time" + httphelpers "git.makerforce.io/photos/photos/internal/httphelpers" + lib "git.makerforce.io/photos/photos/pkg/bucket" "github.com/davidbyttow/govips/pkg/vips" - "github.com/minio/minio-go/v6" ) -var client *http.Client -var minioClient *minio.Client +var client *lib.Client + +type resizeOperation struct { + input []byte + previewOption lib.PreviewOption + output []byte + err error + took time.Duration +} const maxAspectRatio = 6 -var accessKey string -var secretKey string -var endpoint string -var endpointSecure bool - func main() { - // Read configuration - accessKey = os.Getenv("MINIO_ACCESS_KEY") - secretKey = os.Getenv("MINIO_SECRET_KEY") - endpoint = os.Getenv("MINIO_ENDPOINT") - endpointSecure = os.Getenv("MINIO_ENDPOINT_SECURE") == "true" - - // Setup HTTP client - transport := &http.Transport{ - MaxIdleConns: 4, - MaxIdleConnsPerHost: 4, - IdleConnTimeout: 30 * time.Second, - DisableCompression: true, - } - client = &http.Client{ - Transport: transport, - Timeout: 5 * time.Second, - } - // Setup minio client + // Setup bucket client var err error - minioClient, err = minio.New(endpoint, accessKey, secretKey, endpointSecure) + client, err = lib.NewClientFromEnv() if err != nil { panic(err) } + // Setup vips vips.Startup(nil) defer vips.Shutdown() - buf, err := fetch("https://huge.makerforce.io/pub/MVIMG_20200320_154434.jpg") - if err != nil { - panic(err) + server := &http.Server{ + Addr: ":8003", + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, } - - start := time.Now() - resizedBufWEBP, err := resize(buf, vips.ImageTypeWEBP, 320, 70) - end := time.Now() - if err != nil { - panic(err) - } - log.Println("passed", end.Sub(start)) - - start = time.Now() - resizedBufJPEG, err := resize(buf, vips.ImageTypeJPEG, 320, 75) - end = time.Now() - if err != nil { - panic(err) - } - log.Println("passed", end.Sub(start)) - - err = save(resizedBufWEBP, "pub", "out.webp", "image/webp") - if err != nil { - panic(err) - } - - err = save(resizedBufJPEG, "pub", "out.jpg", "image/jpeg") + http.HandleFunc("/update", update) + err = server.ListenAndServe() if err != nil { panic(err) } } -func fetch(url string) ([]byte, error) { - resp, err := client.Get(url) - if err != nil { - return nil, err +func update(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPost { + err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method) + httphelpers.ErrorResponse(w, err) + return } - defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + + bucket := lib.Bucket(req.FormValue("bucket")) + photo := lib.Photo(req.FormValue("photo")) + /* + bucketMetadata, err := client.GetBucketMetadata(bucket) + previewOptions := bucketMetadata.PreviewOptions + */ + previewOptions := lib.DefaultPreviewOptions() + + original, err := client.GetPhoto(bucket, photo) + if err != nil { + httphelpers.ErrorResponse(w, err) + return + } + + resizes := make([]resizeOperation, len(previewOptions)) + for i := range resizes { + resizes[i] = resizeOperation{ + input: original, + previewOption: previewOptions[i], + } + } + + var wg sync.WaitGroup + + for i := range resizes { + op := &resizes[i] + wg.Add(1) + // TODO: timeouts + go func() { + start := time.Now() + log.Printf("Resizing photo %s at h%dq%d", photo, op.previewOption.Height, op.previewOption.Quality) + output, err := resize( + op.input, + op.previewOption.Height, + toVipsImageType(op.previewOption.Format), + op.previewOption.Quality, + ) + op.output = output + op.err = err + end := time.Now() + op.took = end.Sub(start) + wg.Done() + }() + } + + wg.Wait() + + // Check for errors + for _, op := range resizes { + if op.err != nil { + httphelpers.ErrorResponse(w, err) + return + } + } + + // Save + for i := range resizes { + op := &resizes[i] + wg.Add(1) + go func() { + preview := photo.GetPreview(op.previewOption) + err := client.PutPreview(bucket, preview, op.output) + log.Printf("Uploaded photo %s at h%dq%d: %v", photo, op.previewOption.Height, op.previewOption.Quality, err) + op.err = err + wg.Done() + }() + } + + wg.Wait() + + // Check for errors + for _, op := range resizes { + if op.err != nil { + httphelpers.ErrorResponse(w, err) + return + } + } + + timings := make([]time.Duration, 0) + for _, op := range resizes { + timings = append(timings, op.took) + } + w.Header().Add("X-Debug-Resize-Timings", fmt.Sprintf("%v", timings)) } -func resize(buf []byte, format vips.ImageType, height, quality int) ([]byte, error) { +func resize(buf []byte, height int, format vips.ImageType, quality int) ([]byte, error) { i, err := vips.NewImageFromBuffer(buf) if err != nil { return nil, err @@ -110,10 +159,12 @@ func resize(buf []byte, format vips.ImageType, height, quality int) ([]byte, err return outBuf, err } -func save(buf []byte, bucket, key, contentType string) error { - _, err := minioClient.PutObject(bucket, key, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ - ContentType: contentType, - StorageClass: "REDUCED_REDUNDANCY", - }) - return err +func toVipsImageType(format lib.PhotoFormat) vips.ImageType { + if format == lib.PhotoFormatWEBP { + return vips.ImageTypeWEBP + } + if format == lib.PhotoFormatJPEG { + return vips.ImageTypeJPEG + } + return vips.ImageTypeUnknown } diff --git a/internal/httphelpers/error.go b/internal/httphelpers/error.go new file mode 100644 index 0000000..808bccc --- /dev/null +++ b/internal/httphelpers/error.go @@ -0,0 +1,41 @@ +package httphelpers + +import ( + "errors" + "net/http" + + "github.com/minio/minio-go/v6" +) + +var ErrorBadRequest = errors.New("bad request") +var ErrorMethodNotAllowed = errors.New("method not allowed") + +func ErrorResponse(w http.ResponseWriter, err error) { + if err == nil { + panic("Sneaky, you called ErrorResponse without an error. I shall destroy you") + } + errorMessage := err.Error() + errorStatus := http.StatusInternalServerError + + if errors.Is(err, ErrorBadRequest) { + errorStatus = http.StatusBadRequest + } + if errors.Is(err, ErrorMethodNotAllowed) { + errorStatus = http.StatusMethodNotAllowed + } + + var minioErrorResponse minio.ErrorResponse + if errors.As(err, &minioErrorResponse) { + if minioErrorResponse.Code == "NoSuchKey" { + errorStatus = http.StatusNotFound + } + if minioErrorResponse.Code == "NoSuchBucket" { + errorStatus = http.StatusNotFound + } + // Do not handle minio AccessDenied: that's a server error. Our credentials should always be correct + } + + // Let Header.Write() handle escaping + w.Header().Add("X-Debug-Error", errorMessage) + w.WriteHeader(errorStatus) +} diff --git a/pkg/bucket/bucket.go b/pkg/bucket/bucket.go new file mode 100644 index 0000000..2adfa91 --- /dev/null +++ b/pkg/bucket/bucket.go @@ -0,0 +1,28 @@ +package bucket + +import ( + "errors" + "fmt" +) + +type Bucket string + +const bucketMetadataObject = "metadata.json" + +var ErrorInvalidBucket = errors.New("invalid bucket") + +func (b Bucket) Validate() error { + if len(b) < 1 || b == "meta" { + return fmt.Errorf("%w: %v", ErrorInvalidBucket, b) + } + return nil +} + +func (b Bucket) String() string { + return string(b) +} + +type BucketMetadata struct { + Title string `json:"title"` + PreviewOptions []PreviewOption `json:"preview_options"` +} diff --git a/pkg/bucket/client.go b/pkg/bucket/client.go new file mode 100644 index 0000000..734688a --- /dev/null +++ b/pkg/bucket/client.go @@ -0,0 +1,116 @@ +package bucket + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "os" + + "github.com/minio/minio-go/v6" +) + +type Client struct { + *minio.Client +} + +func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Client, error) { + m, err := minio.New(endpoint, accessKey, secretKey, endpointSecure) + if err != nil { + return nil, err + } + + client := &Client{ + Client: m, + } + return client, nil +} + +func NewClientFromEnv() (*Client, error) { + accessKey := os.Getenv("MINIO_ACCESS_KEY") + secretKey := os.Getenv("MINIO_SECRET_KEY") + endpoint := os.Getenv("MINIO_ENDPOINT") + endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true" + return NewClient(endpoint, accessKey, secretKey, endpointSecure) +} + +func (c *Client) GetBucketMetadata(b Bucket) (BucketMetadata, error) { + err := b.Validate() + if err != nil { + return BucketMetadata{}, err + } + + o, err := c.GetObject(b.String(), bucketMetadataObject, minio.GetObjectOptions{}) + if err != nil { + return BucketMetadata{}, err + } + defer o.Close() + buf, err := ioutil.ReadAll(o) + if err != nil { + return BucketMetadata{}, err + } + + m := BucketMetadata{} + err = json.Unmarshal(buf, &m) + if err != nil { + return BucketMetadata{}, err + } + return m, nil +} + +func (c *Client) PutBucketMetadata(b Bucket, m BucketMetadata) error { + err := b.Validate() + if err != nil { + return err + } + + buf, err := json.MarshalIndent(m, "", "\t") + if err != nil { + return err + } + + _, err = c.PutObject(b.String(), bucketMetadataObject, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "application/json", + }) + if err != nil { + return err + } + return nil +} + +func (c *Client) GetPhoto(b Bucket, p Photo) ([]byte, error) { + err := b.Validate() + if err != nil { + return nil, err + } + err = p.Validate() + if err != nil { + return nil, err + } + + o, err := c.GetObject(b.String(), p.String(), minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer o.Close() + return ioutil.ReadAll(o) +} + +func (c *Client) PutPreview(b Bucket, p Preview, buf []byte) error { + err := b.Validate() + if err != nil { + return err + } + err = p.Validate() + if err != nil { + return err + } + + _, err = c.PutObject(b.String(), p.String(), bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: p.Format().ContentType(), + StorageClass: "REDUCED_REDUNDANCY", + }) + if err != nil { + return err + } + return nil +} diff --git a/pkg/bucket/photo.go b/pkg/bucket/photo.go new file mode 100644 index 0000000..cb5c4eb --- /dev/null +++ b/pkg/bucket/photo.go @@ -0,0 +1,24 @@ +package bucket + +import ( + "errors" + "fmt" + "strings" +) + +type Photo string + +const photoPrefix = "photo/" + +var ErrorInvalidPhoto = errors.New("invalid photo") + +func (p Photo) Validate() error { + if !strings.HasPrefix(string(p), photoPrefix) { + return fmt.Errorf("%w: %v", ErrorInvalidPhoto, p) + } + return nil +} + +func (p Photo) String() string { + return string(p) +} diff --git a/pkg/bucket/photopreview.go b/pkg/bucket/photopreview.go new file mode 100644 index 0000000..886eb3b --- /dev/null +++ b/pkg/bucket/photopreview.go @@ -0,0 +1,125 @@ +package bucket + +import ( + "errors" + "fmt" + "mime" + "strings" +) + +type Preview string + +const previewPrefix = "preview/" + +var ErrorInvalidPreview = errors.New("invalid preview") + +func (p Preview) Validate() error { + if !strings.HasPrefix(string(p), previewPrefix) { + return fmt.Errorf("%w: %v", ErrorInvalidPreview, p) + } + return nil +} + +func (p Preview) String() string { + return string(p) +} + +func (p Preview) Format() PhotoFormat { + extIndex := strings.LastIndex(string(p), ".") + return PhotoFormat(mime.TypeByExtension(string(p)[extIndex:])) +} + +type PreviewOption struct { + Height int + //Width int + Format PhotoFormat + Quality int +} + +type PhotoFormat string + +const ( + PhotoFormatJPEG PhotoFormat = "image/jpeg" + PhotoFormatWEBP PhotoFormat = "image/webp" + PhotoFormatHEIF PhotoFormat = "image/heif" + PhotoFormatPNG PhotoFormat = "image/png" + PhotoFormatGIF PhotoFormat = "image/gif" + PhotoFormatTIFF PhotoFormat = "image/tiff" +) + +func (f PhotoFormat) ContentType() string { + return string(f) +} + +var SupportedFormats = []PhotoFormat{ + PhotoFormatJPEG, + PhotoFormatWEBP, + PhotoFormatHEIF, + PhotoFormatPNG, + PhotoFormatGIF, + PhotoFormatTIFF, +} + +type PhotoQualityLevel int + +const ( + PhotoQualityLevelThumbnail PhotoQualityLevel = 0 + PhotoQualityLevelBasic PhotoQualityLevel = 1 + PhotoQualityLevelNormal PhotoQualityLevel = 2 + PhotoQualityLevelExtreme PhotoQualityLevel = 3 +) + +func (f PhotoFormat) Quality(level PhotoQualityLevel) int { + if f == PhotoFormatJPEG || f == PhotoFormatWEBP { + return 34 + int(level)*18 + } + if f == PhotoFormatHEIF { + return 10 + int(level)*10 + } + return 0 +} + +func (p Photo) GetPreview(option PreviewOption) Preview { + objectBase := strings.Replace(string(p), photoPrefix, previewPrefix, 1) + extIndex := strings.LastIndex(objectBase, ".") + base := objectBase[:extIndex] + res := fmt.Sprintf("_h%dq%d", option.Height, option.Quality) + exts, err := mime.ExtensionsByType(option.Format.ContentType()) + if err != nil { + panic(err) + } + if len(exts) < 1 { + panic("missing MIME type to extension mapping, please configure within your operating system") + } + return Preview(base + res + exts[0]) +} + +var defaultSizes = []int{640, 320, 160, 80} +var defaultSizesThumb = []int{120, 60, 30} +var defaultFormats = []PhotoFormat{ + PhotoFormatWEBP, + PhotoFormatJPEG, +} + +func DefaultPreviewOptions() []PreviewOption { + options := make([]PreviewOption, 0) + for _, size := range defaultSizes { + for _, format := range defaultFormats { + options = append(options, PreviewOption{ + Height: size, + Format: format, + Quality: format.Quality(PhotoQualityLevelNormal), + }) + } + } + for _, size := range defaultSizesThumb { + for _, format := range defaultFormats { + options = append(options, PreviewOption{ + Height: size, + Format: format, + Quality: format.Quality(PhotoQualityLevelThumbnail), + }) + } + } + return options +}