From ead40a4ff679446aa14e70407697d68a91e95f60 Mon Sep 17 00:00:00 2001 From: orcas Date: Thu, 20 Jun 2019 13:20:41 +0800 Subject: [PATCH] Basic storage functionality --- .env | 5 +++ README.md | 85 +++++++++++++++++++++++++++++++++++++++- go.mod | 5 +++ go.sum | 17 ++++++++ main.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 70c984c..1f7e494 100644 --- a/.env +++ b/.env @@ -1 +1,6 @@ LISTEN=:80 +MINIO_ENDPOINT=minio:9000 +MINIO_ID=MINIO_ID +MINIO_KEY=MINIO_KEY +MINIO_BUCKET_NAME=beep +MINIO_LOCATION=us-east-1 diff --git a/README.md b/README.md index e95a039..ea864af 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,86 @@ # backend-pictures -Beep backend proxying Minio to act as a fileserver. \ No newline at end of file +Beep backend proxying [Minio](https://min.io) to act as a fileserver. Need a running instance of `minio`. + +**To run this service securely is to run it behind traefik forwarding auth to `backend-auth`** + +## Quickstart + +``` +go build && ./pictures +``` + +## Environment Variables + +Supply environment variables by either exporting them or editing `.env`. + +| ENV | Definition | Default | +| --- | ---------- | ------- | +| LISTEN | Host and port number to listen on | :80 | +| MINIO_ENDPOINT | Host and port of minio | minio:9000 | +| MINIO_ID | Client id to use with minio | MINIO_ID | +| MINIO_KEY | Client key to use with minio | MINIO_KEY | +| MINIO_BUCKET_NAME | Name of bucket to store files in | beep | +| MINIO_LOCATION | Minio bucket region | us-east-1 | + +## API + +All requests need to be passed through `traefik` calling `backend-auth` as Forward Authentication. Otherwise, populate `X-User-Claim` with: + +```json +{ + "userid": "", + "clientid": "" +} +``` + +### Upload FIle + +``` +POST /upload +``` + +Upload a file to be stored. Requires a request of `Content-Type` `multipart/form-data`. + +#### Body + +| Name | Description | +| ---- | ----------- | +| file | File to be uploaded | + +#### Success (200 OK) + +Name of the file as stored. + +#### Errors + +| Code | Description | +| ---- | ----------- | +| 400 | Error parsing file out of request | +| 500 | Error storing file in minio | + +--- + +### Get File + +``` +GET /picture/:filename +``` + +Retrieve a picture by filename. + +#### Params + +| Name | Description | +| ---- | ----------- | +| filename | Name of the file to be retrieved | + +#### Success (200 OK) + +Image file. + +#### Errors + +| Code | Description | +| ---- | ----------- | +| 500 | Error retrieving file from minio | diff --git a/go.mod b/go.mod index a9e778e..f26e860 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,11 @@ module pictures go 1.12 require ( + github.com/go-ini/ini v1.42.0 // indirect github.com/joho/godotenv v1.3.0 github.com/julienschmidt/httprouter v1.2.0 + github.com/minio/minio-go v6.0.14+incompatible + github.com/mitchellh/go-homedir v1.1.0 // indirect + golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 // indirect + golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect ) diff --git a/go.sum b/go.sum index 5b676ac..1cd0492 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,21 @@ +github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= +github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU= +golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 64dfdde..3967afa 100644 --- a/main.go +++ b/main.go @@ -1,16 +1,27 @@ package main import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" "encoding/json" + "io" "log" "net/http" "os" + "strings" "github.com/joho/godotenv" "github.com/julienschmidt/httprouter" + "github.com/minio/minio-go" ) +const MaxBiteSize = 1024 * 1024 * 100 // 100MB + var listen string +var minioClient *minio.Client +var bucketName string func main() { // Load .env @@ -19,11 +30,115 @@ func main() { log.Fatal("Error loading .env file") } listen = os.Getenv("LISTEN") + minioEndpoint := os.Getenv("MINIO_ENDPOINT") + minioID := os.Getenv("MINIO_ID") + minioKey := os.Getenv("MINIO_KEY") + bucketName = os.Getenv("MINIO_BUCKET_NAME") + minioLocation := os.Getenv("MINIO_LOCATION") + + // Minio client + minioClient, err = minio.New(minioEndpoint, minioID, minioKey, false) + if err != nil { + log.Fatal("Error loading minio") + } + + // Create bucket if it doesn't exist + err = minioClient.MakeBucket(bucketName, minioLocation) + if err != nil { + exists, err := minioClient.BucketExists(bucketName) + if err == nil && exists { + log.Printf("Bucket %s already exists", bucketName) + } else { + log.Fatal("Error creating bucket") + } + } else { + log.Printf("Created bucket %s", bucketName) + } // Routes router := httprouter.New() + router.POST("/upload", AuthMiddleware(Upload)) + router.GET("/picture/:filename", GetFile) // Start server log.Printf("starting server on %s", listen) log.Fatal(http.ListenAndServe(listen, router)) } + +// Pull Auth header +type RawClient struct { + UserId string `json:"userid"` + ClientId string `json:"clientid"` +} +func AuthMiddleware(next httprouter.Handle) httprouter.Handle { + return func (w http.ResponseWriter, r *http.Request, p httprouter.Params) { + ua := r.Header.Get("X-User-Claim") + if ua == "" { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + var client RawClient + err := json.Unmarshal([]byte(ua), &client) + + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + context := context.WithValue(r.Context(), "user", client) + next(w, r.WithContext(context), p) + } +} + +func Upload(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + client := r.Context().Value("user").(RawClient) + var buf bytes.Buffer + file, header, err := r.FormFile("file") + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + defer file.Close() + + originalName := strings.Split(header.Filename, ".") + + io.Copy(&buf, file) + reader := bytes.NewReader(buf.Bytes()) + + fileName := RandomHex() + "." + originalName[1] + options := minio.PutObjectOptions{ + UserMetadata: make(map[string] string), + } + options.UserMetadata["owner"] = client.UserId + + _, err = minioClient.PutObject(bucketName, fileName, reader, header.Size, options) + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + io.WriteString(w, fileName) +} + +func RandomHex() string { + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + panic("unable to generate 16 bits of randomness") + } + return hex.EncodeToString(b) +} + +func GetFile(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + fileName := p.ByName("filename") + options := minio.GetObjectOptions{} + + reader, err := minioClient.GetObject(bucketName, fileName, options) + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + io.Copy(w, reader) +}