Implement unauthenticated access control

.gitignore

@@ -1,8 +1,7 @@
 cmd/admin/admin
 cmd/control/control
 cmd/web/web
-cmd/thumbnail/thumbnail
-cmd/indexer/indexer
+cmd/preview/preview
 cmd/proxy/proxy
 
 env

README.md

@@ -3,6 +3,14 @@
 
 A photo bucket management suite.
 
+There are two modes of operation:
+- Domain
+  - Buckets are exactly equal to their domain names
+  - `unset MINIO_DOMAIN`
+- Subdomain
+  - Buckets are named after subdomains
+  - `export MINIO_DOMAIN=your.domain`
+
 ## `admin`
 
 Create new buckets. Standalone tool.
@@ -64,27 +72,20 @@ Generates the web interface for a photo bucket. Also updates the shared asset bu
 
 Regenerate and upload `index.html` and `manage/index.html` to bucket.
 
-## `indexer`
+## `preview`
 
-Pointed to by a reverse proxy to handle the following paths on all buckets:
-
-- `/`
-- `/manage/`
-
-#### `GET /*`
-
-A proxy for all buckets, treats the URL as a directory and serves up directory + `index.html`.
-
-## `thumbnail`
-
-Generate thumbnails from photo buckets. Registers webhooks.
+Generate previews from photo buckets. Registers webhooks.
 
 ### Operations
 
 #### `POST /webhook`
 #### `POST /update?bucket=BUCKET&photo=OBJECT`
 
-1. Perform thumbnail generation using libvips (maybe limit?)
+1. Perform preview generation using libvips (maybe limit?)
 2. Block until done
 
+## `proxy`
+
+Reverse proxies buckets to the minio endpoint, as a substitute for the AWS S3 website hosting features. Serves up `index.html` when URLs end in a slash.
+
 <!-- vim: set conceallevel=2 et ts=2 sw=2: -->

cmd/control/control.go

@@ -3,72 +3,95 @@ package main
 import (
 	"fmt"
 	"net/http"
-	"net/url"
-	"os"
+	"strconv"
 	"time"
 
-	"github.com/minio/minio-go/v6/pkg/s3utils"
-	"github.com/minio/minio-go/v6/pkg/signer"
+	"git.makerforce.io/photos/photos/internal/httphelpers"
+	lib "git.makerforce.io/photos/photos/pkg/bucket"
 )
 
-var accessKey string
-var secretKey string
+var signer *lib.Signer
 
 func main() {
-	// Read configuration
-	accessKey = os.Getenv("MINIO_ACCESS_KEY")
-	secretKey = os.Getenv("MINIO_SECRET_KEY")
+	// Setup bucket signer
+	var err error
+	signer, err = lib.NewSignerFromEnv()
 
 	server := &http.Server{
 		Addr:         ":8000",
 		ReadTimeout:  5 * time.Second,
 		WriteTimeout: 10 * time.Second,
 	}
-	http.HandleFunc("/list", hello)
-	http.HandleFunc("/write", hello2)
-	err := server.ListenAndServe()
+	http.HandleFunc("/list", list)
+	http.HandleFunc("/read", read)
+	http.HandleFunc("/write", write)
+	err = server.ListenAndServe()
 	if err != nil {
 		panic(err)
 	}
 }
 
-func hello(w http.ResponseWriter, req *http.Request) {
-	// Based upon https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L873
-	expires := 60 * time.Second
-	reqValues := make(url.Values)
-	reqValues.Set("list-type", "2")
-	reqValues.Set("metadata", "true")
-	reqValues.Set("encoding-type", "url")
-	reqValues.Set("prefix", "")
-	reqValues.Set("delimiter", "")
-	reqValues.Set("max-keys", fmt.Sprintf("%d", 500))
-	reqValues.Set("start-after", "")
-	// Change path accordingly https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L911
-	// Request URL MUST not have a port
-	reqString := "https://rock.six.six.six.six.six.eurica.eu.org/?" + s3utils.QueryEncode(reqValues)
-	req, err := http.NewRequest("GET", reqString, nil)
-	if err != nil {
-		panic(err)
+func list(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet {
+		err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method)
+		httphelpers.ErrorResponse(w, err)
+		return
 	}
-	signedReq := signer.PreSignV4(*req, accessKey, secretKey, "", "sgp1", int64(expires/time.Second))
 
-	w.Header().Add("Location", signedReq.URL.String())
+	bucket := lib.Bucket(req.FormValue("bucket"))
+	startAfter := req.FormValue("start-after")
+	maxKeysString := req.FormValue("max-keys")
+	if maxKeysString == "" {
+		maxKeysString = "4000"
+	}
+	maxKeys, err := strconv.Atoi(maxKeysString) // Let minio handle error
+
+	url, err := signer.GetBucketPhotos(bucket, startAfter, maxKeys)
+	if err != nil {
+		httphelpers.ErrorResponse(w, err)
+		return
+	}
+
+	w.Header().Add("Location", url)
 	w.WriteHeader(http.StatusTemporaryRedirect)
 }
 
-func hello2(w http.ResponseWriter, req *http.Request) {
-	// Based upon https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L873
-	expires := 60 * time.Second
-	reqValues := make(url.Values)
-	// Change path accordingly https://github.com/minio/minio-go/blob/337bb00bc3c832292b36681c7bde1b56a185c310/api.go#L911
-	// Request URL MUST not have a port
-	reqString := "https://rock.six.six.six.six.six.eurica.eu.org/rock.txt?" + s3utils.QueryEncode(reqValues)
-	req, err := http.NewRequest("PUT", reqString, nil)
-	if err != nil {
-		panic(err)
+func read(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet {
+		err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method)
+		httphelpers.ErrorResponse(w, err)
+		return
 	}
-	signedReq := signer.PreSignV4(*req, accessKey, secretKey, "", "sgp1", int64(expires/time.Second))
 
-	w.Header().Add("Location", signedReq.URL.String())
+	bucket := lib.Bucket(req.FormValue("bucket"))
+	photo := lib.Photo(req.FormValue("photo"))
+
+	url, err := signer.GetPhoto(bucket, photo)
+	if err != nil {
+		httphelpers.ErrorResponse(w, err)
+		return
+	}
+
+	w.Header().Add("Location", url)
+	w.WriteHeader(http.StatusTemporaryRedirect)
+}
+
+func write(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodPut {
+		err := fmt.Errorf("%w: %v", httphelpers.ErrorMethodNotAllowed, req.Method)
+		httphelpers.ErrorResponse(w, err)
+		return
+	}
+
+	bucket := lib.Bucket(req.FormValue("bucket"))
+	photo := lib.Photo(req.FormValue("photo"))
+
+	url, err := signer.PutPhoto(bucket, photo)
+	if err != nil {
+		httphelpers.ErrorResponse(w, err)
+		return
+	}
+
+	w.Header().Add("Location", url)
 	w.WriteHeader(http.StatusTemporaryRedirect)
 }

cmd/preview/preview.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"time"
 
-	httphelpers "git.makerforce.io/photos/photos/internal/httphelpers"
+	"git.makerforce.io/photos/photos/internal/httphelpers"
 	lib "git.makerforce.io/photos/photos/pkg/bucket"
 	"github.com/davidbyttow/govips/pkg/vips"
 )

cmd/proxy/proxy.go

@@ -10,17 +10,20 @@ import (
 	"strings"
 	"time"
 
+	internal_s3utils "git.makerforce.io/photos/photos/internal/s3utils"
 	"github.com/miekg/dns"
 )
 
 var endpoint string
 var endpointSecure bool
+var domain string
 var behindProxy bool
 
 func main() {
 	// Read configuration
 	endpoint = os.Getenv("MINIO_ENDPOINT")
 	endpointSecure = os.Getenv("MINIO_ENDPOINT_SECURE") == "true"
+	domain = os.Getenv("MINIO_DOMAIN")
 	behindProxy = os.Getenv("BEHIND_PROXY") == "true"
 
 	server := &http.Server{
@@ -53,7 +56,19 @@ func director(req *http.Request) {
 		req.URL.Scheme = "https"
 	}
 	req.URL.Host = endpoint
-	req.URL.Path = mapPath(req.URL.Path, host)
+	if len(domain) < 1 {
+		// If we are using domains, rewrite the host into path
+		req.URL.Path = internal_s3utils.PathFromHost(mapPath(*req), host)
+	} else {
+		// If we are using subdomains, set the host header
+		req.URL.Path = mapPath(*req)
+		//req.Header.Set("Host", host)
+	}
+
+	// Prevent MINIO from issuing redirects
+	userAgent := req.Header.Get("User-Agent")
+	userAgent = strings.ReplaceAll(userAgent, "Mozilla", "M-o-z-i-l-l-a")
+	req.Header.Set("User-Agent", userAgent)
 
 	if !behindProxy {
 		// Clear existing unsafe headers
@@ -66,12 +81,16 @@ func director(req *http.Request) {
 		req.Header.Set("X-Forwarded-Port", port)
 	}
 
-	log.Println(req.URL)
+	log.Println(req.Header.Get("Host"), req.URL)
 }
 
-func mapPath(path, host string) string {
+func mapPath(req http.Request) string {
+	path := req.URL.Path
+	if req.FormValue("list-type") == "2" {
+		return path
+	}
 	if strings.HasSuffix(path, "/") {
 		path += "index.html"
 	}
-	return "/" + host + path
+	return path
 }

internal/s3utils/hostpath.go

@@ -0,0 +1,8 @@
+package s3utils
+
+func PathFromHost(path, host string) string {
+	if len(host) > 0 {
+		return "/" + host + path
+	}
+	return path
+}

pkg/bucket/client.go

@@ -13,8 +13,8 @@ type Client struct {
 	*minio.Client
 }
 
-func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Client, error) {
-	m, err := minio.New(endpoint, accessKey, secretKey, endpointSecure)
+func NewClient(endpoint, accessKey, secretKey, regionName string, endpointSecure bool) (*Client, error) {
+	m, err := minio.NewWithRegion(endpoint, accessKey, secretKey, endpointSecure, regionName)
 	if err != nil {
 		return nil, err
 	}
@@ -28,9 +28,10 @@ func NewClient(endpoint, accessKey, secretKey string, endpointSecure bool) (*Cli
 func NewClientFromEnv() (*Client, error) {
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
 	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	regionName := os.Getenv("MINIO_REGION_NAME")
 	endpoint := os.Getenv("MINIO_ENDPOINT")
 	endpointSecure := os.Getenv("MINIO_ENDPOINT_SECURE") == "true"
-	return NewClient(endpoint, accessKey, secretKey, endpointSecure)
+	return NewClient(endpoint, accessKey, secretKey, regionName, endpointSecure)
 }
 
 func (c *Client) GetBucketMetadata(b Bucket) (BucketMetadata, error) {

pkg/bucket/photo.go

@@ -22,3 +22,7 @@ func (p Photo) Validate() error {
 func (p Photo) String() string {
 	return string(p)
 }
+
+func (p Photo) Path() string {
+	return "/" + string(p)
+}

pkg/bucket/signer.go

@@ -0,0 +1,176 @@
+package bucket
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"time"
+
+	internal_s3utils "git.makerforce.io/photos/photos/internal/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/signer"
+)
+
+type Signer struct {
+	accessKey    string
+	secretKey    string
+	regionName   string
+	bucketSecure bool
+	expirations  Expirations
+}
+
+type Expirations struct {
+	// Expiration time for list and read in time.Duration
+	Read time.Duration
+	// Expiration time for write in time.Duration
+	Write time.Duration
+}
+
+var ErrorExpirationTooLow = errors.New("expiration time too low")
+
+func NewSigner(accessKey, secretKey, regionName string, bucketSecure bool, expirations Expirations) (*Signer, error) {
+	if expirations.Read == 0 {
+		expirations.Read = 30 * time.Minute
+	}
+	if expirations.Write == 0 {
+		expirations.Write = 5 * time.Minute
+	}
+	if expirations.Read < time.Second || expirations.Write < time.Second {
+		return nil, ErrorExpirationTooLow
+	}
+	signer := &Signer{
+		accessKey:    accessKey,
+		secretKey:    secretKey,
+		regionName:   regionName,
+		bucketSecure: bucketSecure,
+		expirations:  expirations,
+	}
+	return signer, nil
+}
+
+func NewSignerFromEnv() (*Signer, error) {
+	accessKey := os.Getenv("MINIO_ACCESS_KEY")
+	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	regionName := os.Getenv("MINIO_REGION_NAME")
+	bucketSecure := os.Getenv("BUCKET_SECURE") == "true"
+	expirationRead, _ := strconv.Atoi(os.Getenv("EXPIRATION_READ"))
+	expirationWrite, _ := strconv.Atoi(os.Getenv("EXPIRATION_WRITE"))
+	expirations := Expirations{Read: time.Duration(expirationRead), Write: time.Duration(expirationWrite)}
+	return NewSigner(accessKey, secretKey, regionName, bucketSecure, expirations)
+}
+
+func (s *Signer) baseBucket(b Bucket) url.URL {
+	url := url.URL{
+		Scheme: "http",
+		Host:   b.String(),
+		Path:   "",
+	}
+	if s.bucketSecure {
+		url.Scheme = "https"
+	}
+	return url
+}
+
+func (s *Signer) preSignV4(req http.Request) http.Request {
+	domain := os.Getenv("MINIO_DOMAIN")
+
+	// Validate host
+	host, _, err := net.SplitHostPort(req.Host)
+	if err != nil {
+		host = req.Host
+	}
+	originalPath := req.URL.Path
+
+	if len(domain) < 1 {
+		// If we are using domains, rewrite the host into path
+		req.URL.Path = internal_s3utils.PathFromHost(req.URL.Path, host)
+	}
+	// If we are using subdomains, sign the direct URL
+	signedReq := signer.PreSignV4(
+		req,
+		s.accessKey, s.secretKey, "",
+		"sgp1",
+		int64(s.expirations.Read/time.Second),
+	)
+	signedReq.URL.Path = originalPath
+	return *signedReq
+}
+
+func listBucketParams(prefix, startAfter string, maxKeys int) url.Values {
+	params := make(url.Values)
+	params.Set("list-type", "2")
+	params.Set("metadata", "true")
+	params.Set("encoding-type", "url")
+	params.Set("prefix", prefix)
+	params.Set("delimiter", "")
+	params.Set("max-keys", strconv.Itoa(maxKeys))
+	params.Set("start-after", startAfter)
+	return params
+}
+
+func (s *Signer) GetBucketPhotos(b Bucket, startAfter string, maxKeys int) (string, error) {
+	err := b.Validate()
+	if err != nil {
+		return "", err
+	}
+
+	url := s.baseBucket(b)
+	url.Path += "/"
+	params := listBucketParams(photoPrefix, startAfter, maxKeys)
+	url.RawQuery = s3utils.QueryEncode(params)
+
+	req, err := http.NewRequest("GET", url.String(), nil)
+	if err != nil {
+		return "", err
+	}
+
+	signedReq := s.preSignV4(*req)
+	return signedReq.URL.String(), nil
+}
+
+func (s *Signer) GetPhoto(b Bucket, p Photo) (string, error) {
+	err := b.Validate()
+	if err != nil {
+		return "", err
+	}
+	err = p.Validate()
+	if err != nil {
+		return "", err
+	}
+
+	url := s.baseBucket(b)
+	url.Path += p.Path()
+
+	req, err := http.NewRequest("GET", url.String(), nil)
+	if err != nil {
+		return "", err
+	}
+
+	signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Read/time.Second))
+	return signedReq.URL.String(), nil
+}
+
+func (s *Signer) PutPhoto(b Bucket, p Photo) (string, error) {
+	err := b.Validate()
+	if err != nil {
+		return "", err
+	}
+	err = p.Validate()
+	if err != nil {
+		return "", err
+	}
+
+	url := s.baseBucket(b)
+	url.Path += p.Path()
+
+	req, err := http.NewRequest("PUT", url.String(), nil)
+	if err != nil {
+		return "", err
+	}
+
+	signedReq := signer.PreSignV4(*req, s.accessKey, s.secretKey, "", "sgp1", int64(s.expirations.Write/time.Second))
+	return signedReq.URL.String(), nil
+}