Build out thumbnail generation API
parent
bbe387a1b6
commit
39188008b1
GPG Key ID:
BC367D33F140B5C2
|
|
@ -3,3 +3,5 @@ cmd/control/control
|
|||
cmd/web/web
|
||||
cmd/indexer/indexer
|
||||
cmd/thumbnail/thumbnail
|
||||
|
||||
env
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Implement access controls by signing or proxying requests.
|
|||
#### `GET /list?bucket=BUCKET&auth=TOKEN`
|
||||
|
||||
1. Consult the bucket for metadata.json
|
||||
2. Get read access method for the bucket
|
||||
2. Get list access method for the bucket
|
||||
3. Validate the token against the access method
|
||||
4. Return ListObjectsV2 for prefix `photo/`
|
||||
- Can also 307 redirect to the bucket read URL, if is public readable
|
||||
|
|
@ -82,9 +82,9 @@ Generate thumbnails from photo buckets. Registers webhooks.
|
|||
### Operations
|
||||
|
||||
#### `POST /webhook`
|
||||
#### `POST /update?bucket=BUCKET&object=OBJECT`
|
||||
#### `POST /update?bucket=BUCKET&photo=OBJECT`
|
||||
|
||||
1. Perform thumbnail generation using libvips in a pool queue.
|
||||
1. Perform thumbnail generation using libvips (maybe limit?)
|
||||
2. Block until done
|
||||
|
||||
<!-- vim: set conceallevel=2 et ts=2 sw=2: -->
|
||||
|
|
|
|||
|
|
@ -20,12 +20,16 @@ func main() {
|
|||
secretKey = os.Getenv("MINIO_SECRET_KEY")
|
||||
|
||||
server := &http.Server{
|
||||
Addr: ":8000",
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
http.HandleFunc("/list", hello)
|
||||
http.HandleFunc("/write", hello2)
|
||||
server.ListenAndServe()
|
||||
err := server.ListenAndServe()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func hello(w http.ResponseWriter, req *http.Request) {
|
||||
|
|
|
|||
|
|
@ -1,97 +1,146 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
httphelpers "git.makerforce.io/photos/photos/internal/httphelpers"
|
||||
lib "git.makerforce.io/photos/photos/pkg/bucket"
|
||||
"github.com/davidbyttow/govips/pkg/vips"
|
||||
"github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
var client *http.Client
|
||||
var minioClient *minio.Client
|
||||
var client *lib.Client
|
||||
|
||||
type resizeOperation struct {
|
||||
input []byte
|
||||
previewOption lib.PreviewOption
|
||||
output []byte
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
const maxAspectRatio = 6
|
||||
|
||||
var accessKey string
|
||||
var secretKey string
|
||||
var endpoint string
|
||||
var endpointSecure bool
|
||||
|
||||
func main() {
|
||||
// Read configuration
|
||||
accessKey = os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey = os.Getenv("MINIO_SECRET_KEY")
|
||||
endpoint = os.Getenv("MINIO_ENDPOINT")
|
||||
endpointSecure = os.Getenv("MINIO_ENDPOINT_SECURE") == "true"
|
||||
|
||||
// Setup HTTP client
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 4,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
DisableCompression: true,
|
||||
}
|
||||
client = &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
// Setup minio client
|
||||
// Setup bucket client
|
||||
var err error
|
||||
minioClient, err = minio.New(endpoint, accessKey, secretKey, endpointSecure)
|
||||
client, err = lib.NewClientFromEnv()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Setup vips
|
||||
vips.Startup(nil)
|
||||
defer vips.Shutdown()
|
||||
|
||||
buf, err := fetch("https://huge.makerforce.io/pub/MVIMG_20200320_154434.jpg")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
server := &http.Server{
|
||||
Addr: ":8003",
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
resizedBufWEBP, err := resize(buf, vips.ImageTypeWEBP, 320, 70)
|
||||
end := time.Now()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Println("passed", end.Sub(start))
|
||||
|
||||
start = time.Now()
|
||||
resizedBufJPEG, err := resize(buf, vips.ImageTypeJPEG, 320, 75)
|
||||
end = time.Now()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Println("passed", end.Sub(start))
|
||||
|
||||
err = save(resizedBufWEBP, "pub", "out.webp", "image/webp")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = save(resizedBufJPEG, "pub", "out.jpg", "image/jpeg")
|
||||
http.HandleFunc("/update", update)
|
||||
err = server.ListenAndServe()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fetch(url string) ([]byte, error) {
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func update(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodPost {
|
||||
err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method)
|
||||
httphelpers.ErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return ioutil.ReadAll(resp.Body)
|
||||
|
||||
bucket := lib.Bucket(req.FormValue("bucket"))
|
||||
photo := lib.Photo(req.FormValue("photo"))
|
||||
/*
|
||||
bucketMetadata, err := client.GetBucketMetadata(bucket)
|
||||
previewOptions := bucketMetadata.PreviewOptions
|
||||
*/
|
||||
previewOptions := lib.DefaultPreviewOptions()
|
||||
|
||||
original, err := client.GetPhoto(bucket, photo)
|
||||
if err != nil {
|
||||
httphelpers.ErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
resizes := make([]resizeOperation, len(previewOptions))
|
||||
for i := range resizes {
|
||||
resizes[i] = resizeOperation{
|
||||
input: original,
|
||||
previewOption: previewOptions[i],
|
||||
}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range resizes {
|
||||
op := &resizes[i]
|
||||
wg.Add(1)
|
||||
// TODO: timeouts
|
||||
go func() {
|
||||
start := time.Now()
|
||||
log.Printf("Resizing photo %s at h%dq%d", photo, op.previewOption.Height, op.previewOption.Quality)
|
||||
output, err := resize(
|
||||
op.input,
|
||||
op.previewOption.Height,
|
||||
toVipsImageType(op.previewOption.Format),
|
||||
op.previewOption.Quality,
|
||||
)
|
||||
op.output = output
|
||||
op.err = err
|
||||
end := time.Now()
|
||||
op.took = end.Sub(start)
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Check for errors
|
||||
for _, op := range resizes {
|
||||
if op.err != nil {
|
||||
httphelpers.ErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Save
|
||||
for i := range resizes {
|
||||
op := &resizes[i]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
preview := photo.GetPreview(op.previewOption)
|
||||
err := client.PutPreview(bucket, preview, op.output)
|
||||
log.Printf("Uploaded photo %s at h%dq%d: %v", photo, op.previewOption.Height, op.previewOption.Quality, err)
|
||||
op.err = err
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Check for errors
|
||||
for _, op := range resizes {
|
||||
if op.err != nil {
|
||||
httphelpers.ErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
timings := make([]time.Duration, 0)
|
||||
for _, op := range resizes {
|
||||
timings = append(timings, op.took)
|
||||
}
|
||||
w.Header().Add("X-Debug-Resize-Timings", fmt.Sprintf("%v", timings))
|
||||
}
|
||||
|
||||
func resize(buf []byte, format vips.ImageType, height, quality int) ([]byte, error) {
|
||||
func resize(buf []byte, height int, format vips.ImageType, quality int) ([]byte, error) {
|
||||
i, err := vips.NewImageFromBuffer(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -110,10 +159,12 @@ func resize(buf []byte, format vips.ImageType, height, quality int) ([]byte, err
|
|||
return outBuf, err
|
||||
}
|
||||
|
||||
func save(buf []byte, bucket, key, contentType string) error {
|
||||
_, err := minioClient.PutObject(bucket, key, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
StorageClass: "REDUCED_REDUNDANCY",
|
||||
})
|
||||
return err
|
||||
func toVipsImageType(format lib.PhotoFormat) vips.ImageType {
|
||||
if format == lib.PhotoFormatWEBP {
|
||||
return vips.ImageTypeWEBP
|
||||
}
|
||||
if format == lib.PhotoFormatJPEG {
|
||||
return vips.ImageTypeJPEG
|
||||
}
|
||||
return vips.ImageTypeUnknown
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
package httphelpers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
var ErrorBadRequest = errors.New("bad request")
|
||||
var ErrorMethodNotAllowed = errors.New("method not allowed")
|
||||
|
||||
func ErrorResponse(w http.ResponseWriter, err error) {
|
||||
if err == nil {
|
||||
panic("Sneaky, you called ErrorResponse without an error. I shall destroy you")
|
||||
}
|
||||
errorMessage := err.Error()
|
||||
errorStatus := http.StatusInternalServerError
|
||||
|
||||
if errors.Is(err, ErrorBadRequest) {
|
||||
errorStatus = http.StatusBadRequest
|
||||
}
|
||||
if errors.Is(err, ErrorMethodNotAllowed) {
|
||||
errorStatus = http.StatusMethodNotAllowed
|
||||
}
|
||||
|
||||
var minioErrorResponse minio.ErrorResponse
|
||||
if errors.As(err, &minioErrorResponse) {
|
||||
if minioErrorResponse.Code == "NoSuchKey" {
|
||||
errorStatus = http.StatusNotFound
|
||||
}
|
||||
if minioErrorResponse.Code == "NoSuchBucket" {
|
||||
errorStatus = http.StatusNotFound
|
||||
}
|
||||
// Do not handle minio AccessDenied: that's a server error. Our credentials should always be correct
|
||||
}
|
||||
|
||||
// Let Header.Write() handle escaping
|
||||
w.Header().Add("X-Debug-Error", errorMessage)
|
||||
w.WriteHeader(errorStatus)
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package bucket
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type Bucket string
|
||||
|
||||
const bucketMetadataObject = "metadata.json"
|
||||
|
||||
var ErrorInvalidBucket = errors.New("invalid bucket")
|
||||
|
||||
func (b Bucket) Validate() error {
|
||||
if len(b) < 1 || b == "meta" {
|
||||
return fmt.Errorf("%w: %v", ErrorInvalidBucket, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b Bucket) String() string {
|
||||
return string(b)
|
||||
}
|
||||
|
||||
type BucketMetadata struct {
|
||||
Title string `json:"title"`
|
||||
PreviewOptions []PreviewOption `json:"preview_options"`
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
package bucket
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
*minio.Client
|
||||
}
|
||||
|
||||
func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Client, error) {
|
||||
m, err := minio.New(endpoint, accessKey, secretKey, endpointSecure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := &Client{
|
||||
Client: m,
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func NewClientFromEnv() (*Client, error) {
|
||||
accessKey := os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey := os.Getenv("MINIO_SECRET_KEY")
|
||||
endpoint := os.Getenv("MINIO_ENDPOINT")
|
||||
endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true"
|
||||
return NewClient(endpoint, accessKey, secretKey, endpointSecure)
|
||||
}
|
||||
|
||||
func (c *Client) GetBucketMetadata(b Bucket) (BucketMetadata, error) {
|
||||
err := b.Validate()
|
||||
if err != nil {
|
||||
return BucketMetadata{}, err
|
||||
}
|
||||
|
||||
o, err := c.GetObject(b.String(), bucketMetadataObject, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return BucketMetadata{}, err
|
||||
}
|
||||
defer o.Close()
|
||||
buf, err := ioutil.ReadAll(o)
|
||||
if err != nil {
|
||||
return BucketMetadata{}, err
|
||||
}
|
||||
|
||||
m := BucketMetadata{}
|
||||
err = json.Unmarshal(buf, &m)
|
||||
if err != nil {
|
||||
return BucketMetadata{}, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (c *Client) PutBucketMetadata(b Bucket, m BucketMetadata) error {
|
||||
err := b.Validate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf, err := json.MarshalIndent(m, "", "\t")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = c.PutObject(b.String(), bucketMetadataObject, bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{
|
||||
ContentType: "application/json",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) GetPhoto(b Bucket, p Photo) ([]byte, error) {
|
||||
err := b.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = p.Validate()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
o, err := c.GetObject(b.String(), p.String(), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer o.Close()
|
||||
return ioutil.ReadAll(o)
|
||||
}
|
||||
|
||||
func (c *Client) PutPreview(b Bucket, p Preview, buf []byte) error {
|
||||
err := b.Validate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = p.Validate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = c.PutObject(b.String(), p.String(), bytes.NewBuffer(buf), int64(len(buf)), minio.PutObjectOptions{
|
||||
ContentType: p.Format().ContentType(),
|
||||
StorageClass: "REDUCED_REDUNDANCY",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package bucket
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Photo string
|
||||
|
||||
const photoPrefix = "photo/"
|
||||
|
||||
var ErrorInvalidPhoto = errors.New("invalid photo")
|
||||
|
||||
func (p Photo) Validate() error {
|
||||
if !strings.HasPrefix(string(p), photoPrefix) {
|
||||
return fmt.Errorf("%w: %v", ErrorInvalidPhoto, p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p Photo) String() string {
|
||||
return string(p)
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
package bucket
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Preview string
|
||||
|
||||
const previewPrefix = "preview/"
|
||||
|
||||
var ErrorInvalidPreview = errors.New("invalid preview")
|
||||
|
||||
func (p Preview) Validate() error {
|
||||
if !strings.HasPrefix(string(p), previewPrefix) {
|
||||
return fmt.Errorf("%w: %v", ErrorInvalidPreview, p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p Preview) String() string {
|
||||
return string(p)
|
||||
}
|
||||
|
||||
func (p Preview) Format() PhotoFormat {
|
||||
extIndex := strings.LastIndex(string(p), ".")
|
||||
return PhotoFormat(mime.TypeByExtension(string(p)[extIndex:]))
|
||||
}
|
||||
|
||||
type PreviewOption struct {
|
||||
Height int
|
||||
//Width int
|
||||
Format PhotoFormat
|
||||
Quality int
|
||||
}
|
||||
|
||||
type PhotoFormat string
|
||||
|
||||
const (
|
||||
PhotoFormatJPEG PhotoFormat = "image/jpeg"
|
||||
PhotoFormatWEBP PhotoFormat = "image/webp"
|
||||
PhotoFormatHEIF PhotoFormat = "image/heif"
|
||||
PhotoFormatPNG PhotoFormat = "image/png"
|
||||
PhotoFormatGIF PhotoFormat = "image/gif"
|
||||
PhotoFormatTIFF PhotoFormat = "image/tiff"
|
||||
)
|
||||
|
||||
func (f PhotoFormat) ContentType() string {
|
||||
return string(f)
|
||||
}
|
||||
|
||||
var SupportedFormats = []PhotoFormat{
|
||||
PhotoFormatJPEG,
|
||||
PhotoFormatWEBP,
|
||||
PhotoFormatHEIF,
|
||||
PhotoFormatPNG,
|
||||
PhotoFormatGIF,
|
||||
PhotoFormatTIFF,
|
||||
}
|
||||
|
||||
type PhotoQualityLevel int
|
||||
|
||||
const (
|
||||
PhotoQualityLevelThumbnail PhotoQualityLevel = 0
|
||||
PhotoQualityLevelBasic PhotoQualityLevel = 1
|
||||
PhotoQualityLevelNormal PhotoQualityLevel = 2
|
||||
PhotoQualityLevelExtreme PhotoQualityLevel = 3
|
||||
)
|
||||
|
||||
func (f PhotoFormat) Quality(level PhotoQualityLevel) int {
|
||||
if f == PhotoFormatJPEG || f == PhotoFormatWEBP {
|
||||
return 34 + int(level)*18
|
||||
}
|
||||
if f == PhotoFormatHEIF {
|
||||
return 10 + int(level)*10
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (p Photo) GetPreview(option PreviewOption) Preview {
|
||||
objectBase := strings.Replace(string(p), photoPrefix, previewPrefix, 1)
|
||||
extIndex := strings.LastIndex(objectBase, ".")
|
||||
base := objectBase[:extIndex]
|
||||
res := fmt.Sprintf("_h%dq%d", option.Height, option.Quality)
|
||||
exts, err := mime.ExtensionsByType(option.Format.ContentType())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if len(exts) < 1 {
|
||||
panic("missing MIME type to extension mapping, please configure within your operating system")
|
||||
}
|
||||
return Preview(base + res + exts[0])
|
||||
}
|
||||
|
||||
var defaultSizes = []int{640, 320, 160, 80}
|
||||
var defaultSizesThumb = []int{120, 60, 30}
|
||||
var defaultFormats = []PhotoFormat{
|
||||
PhotoFormatWEBP,
|
||||
PhotoFormatJPEG,
|
||||
}
|
||||
|
||||
func DefaultPreviewOptions() []PreviewOption {
|
||||
options := make([]PreviewOption, 0)
|
||||
for _, size := range defaultSizes {
|
||||
for _, format := range defaultFormats {
|
||||
options = append(options, PreviewOption{
|
||||
Height: size,
|
||||
Format: format,
|
||||
Quality: format.Quality(PhotoQualityLevelNormal),
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, size := range defaultSizesThumb {
|
||||
for _, format := range defaultFormats {
|
||||
options = append(options, PreviewOption{
|
||||
Height: size,
|
||||
Format: format,
|
||||
Quality: format.Quality(PhotoQualityLevelThumbnail),
|
||||
})
|
||||
}
|
||||
}
|
||||
return options
|
||||
}
|
||||
Loading…
Reference in New Issue